From 387e713f8ff69c79d6949d84b86f23b6280aa800 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 11 Mar 2023 15:45:49 +1030 Subject: [PATCH 001/129] Update dev CI to use v1.09 --- .github/actions/DownloadFBuild/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/DownloadFBuild/action.yml b/.github/actions/DownloadFBuild/action.yml index 021f33777..dbd0d17be 100644 --- a/.github/actions/DownloadFBuild/action.yml +++ b/.github/actions/DownloadFBuild/action.yml @@ -5,7 +5,7 @@ inputs: version: description: Version to download required: false - default: 1.06 + default: 1.09 runs: using: composite From 3d4ec0e420eae1ccf8d0c0e1b5f29c54b1d61f98 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 11 Mar 2023 15:48:14 +1030 Subject: [PATCH 002/129] Update OSX CI to use macos11 - github reports that 10.15 is deprecated --- .github/workflows/OSX.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/OSX.yml b/.github/workflows/OSX.yml index 3e1f7618d..75b7ce2b1 100644 --- a/.github/workflows/OSX.yml +++ b/.github/workflows/OSX.yml @@ -14,7 +14,7 @@ jobs: matrix: include: - cfg: Build & Tests - os: macos-10.15 + os: macos-11 name: OSX (${{ matrix.cfg }}) runs-on: ${{ matrix.os }} From 4c876e6d59576509756b9b895df28ccfd98135d1 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 11 Mar 2023 15:59:23 +1030 Subject: [PATCH 003/129] Switch CI "checkout" action to v3 - update to avoid deprecation warnings about the old action using nodejs 12 --- .github/workflows/Linux.yml | 2 +- .github/workflows/OSX.yml | 2 +- .github/workflows/Windows.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index 27cd0109c..c030e870b 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -41,7 +41,7 @@ jobs: continue-on-error: ${{ matrix.can-fail || false }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/DownloadFBuild diff --git a/.github/workflows/OSX.yml b/.github/workflows/OSX.yml index 75b7ce2b1..5c6b2d3a9 100644 --- a/.github/workflows/OSX.yml +++ b/.github/workflows/OSX.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/DownloadFBuild diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml index 22b80e9da..41f52d23a 100644 --- a/.github/workflows/Windows.yml +++ b/.github/workflows/Windows.yml @@ -27,7 +27,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/DownloadFBuild From f675ad9100cff8b970b5d9632cdc7fb6bd22afc2 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 11 Mar 2023 16:07:03 +1030 Subject: [PATCH 004/129] Add VS2022 to CI coverage --- .github/workflows/Windows.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml index 41f52d23a..a61ff09c0 100644 --- a/.github/workflows/Windows.yml +++ b/.github/workflows/Windows.yml @@ -13,6 +13,9 @@ jobs: fail-fast: false matrix: include: + - cfg: Build & Tests + vs: 2022 + os: windows-2019 - cfg: Build & Tests vs: 2019 os: windows-2019 @@ -21,7 +24,7 @@ jobs: os: windows-2019 - cfg: Analyze vs: 2019 - os: windows-2019 + os: windows-2022 name: Windows (${{ matrix.cfg }}, VS ${{ matrix.vs }}) runs-on: ${{ matrix.os }} From f3fe70e819b4da35d3aae71755d6ca0ce0bca8e4 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 11 Mar 2023 16:10:46 +1030 Subject: [PATCH 005/129] Remove VS2022 from CI coverage - some configuration issues need to be resolved --- .github/workflows/Windows.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml index a61ff09c0..41f52d23a 100644 --- a/.github/workflows/Windows.yml +++ b/.github/workflows/Windows.yml @@ -13,9 +13,6 @@ jobs: fail-fast: false matrix: include: - - cfg: Build & Tests - vs: 2022 - os: windows-2019 - cfg: Build & Tests vs: 2019 os: windows-2019 @@ -24,7 +21,7 @@ jobs: os: windows-2019 - cfg: Analyze vs: 2019 - os: windows-2022 + os: windows-2019 name: Windows (${{ matrix.cfg }}, VS ${{ matrix.vs }}) runs-on: ${{ matrix.os }} From f6f3ea27aae09e7d2d347e6ecf0bfd96c8a8dd46 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 12 Mar 2023 15:05:53 +1030 Subject: [PATCH 006/129] Add xxHash v0.8.1 (not yet used) --- External/xxHash/0.8.1/CHANGELOG | 71 + External/xxHash/0.8.1/LICENSE | 26 + External/xxHash/0.8.1/README.md | 236 + External/xxHash/0.8.1/libxxhash.pc.in | 15 + External/xxHash/0.8.1/xxh3.h | 55 + External/xxHash/0.8.1/xxh_x86dispatch.c | 770 ++++ External/xxHash/0.8.1/xxh_x86dispatch.h | 86 + External/xxHash/0.8.1/xxhash.c | 43 + External/xxHash/0.8.1/xxhash.h | 5580 +++++++++++++++++++++++ 9 files changed, 6882 insertions(+) create mode 100644 External/xxHash/0.8.1/CHANGELOG create mode 100644 External/xxHash/0.8.1/LICENSE create mode 100644 External/xxHash/0.8.1/README.md create mode 100644 External/xxHash/0.8.1/libxxhash.pc.in create mode 100644 External/xxHash/0.8.1/xxh3.h create mode 100644 External/xxHash/0.8.1/xxh_x86dispatch.c create mode 100644 External/xxHash/0.8.1/xxh_x86dispatch.h create mode 100644 External/xxHash/0.8.1/xxhash.c create mode 100644 External/xxHash/0.8.1/xxhash.h diff --git a/External/xxHash/0.8.1/CHANGELOG b/External/xxHash/0.8.1/CHANGELOG new file mode 100644 index 000000000..ff59d8bb1 --- /dev/null +++ b/External/xxHash/0.8.1/CHANGELOG @@ -0,0 +1,71 @@ +v0.8.1 +- perf : much improved performance for XXH3 streaming variants, notably on gcc and msvc +- perf : improved XXH64 speed and latency on small inputs +- perf : small XXH32 speed and latency improvement on small inputs of random size +- perf : minor stack usage improvement for XXH32 and XXH64 +- api : new experimental variants XXH3_*_withSecretandSeed() +- api : update XXH3_generateSecret(), can no generate secret of any size (>= XXH3_SECRET_SIZE_MIN) +- cli : xxhsum can now generate and check XXH3 checksums, using command `-H3` +- build: can build xxhash without XXH3, with new build macro XXH_NO_XXH3 +- build: fix xxh_x86dispatch build with MSVC, by @apankrat +- build: XXH_INLINE_ALL can always be used safely, even after XXH_NAMESPACE or a previous XXH_INLINE_ALL +- build: improved PPC64LE vector support, by @mpe +- install: fix pkgconfig, by @ellert +- install: compatibility with Haiku, by @Begasus +- doc : code comments made compatible with doxygen, by @easyaspi314 +- misc : XXH_ACCEPT_NULL_INPUT_POINTER is no longer necessary, all functions can accept NULL input pointers, as long as size == 0 +- misc : complete refactor of CI tests on Github Actions, offering much larger coverage, by @t-mat +- misc : xxhsum code base split into multiple specialized units, within directory cli/, by @easyaspi314 + +v0.8.0 +- api : stabilize XXH3 +- cli : xxhsum can parse BSD-style --check lines, by @WayneD +- cli : `xxhsum -` accepts console input, requested by @jaki +- cli : xxhsum accepts -- separator, by @jaki +- cli : fix : print correct default algo for symlinked helpers, by @martinetd +- install: improved pkgconfig script, allowing custom install locations, requested by @ellert + +v0.7.4 +- perf: automatic vector detection and selection at runtime (`xxh_x86dispatch.h`), initiated by @easyaspi314 +- perf: added AVX512 support, by @gzm55 +- api : new: secret generator `XXH_generateSecret()`, suggested by @koraa +- api : fix: XXH3_state_t is movable, identified by @koraa +- api : fix: state is correctly aligned in AVX mode (unlike `malloc()`), by @easyaspi314 +- api : fix: streaming generated wrong values in some combination of random ingestion lengths, reported by @WayneD +- cli : fix unicode print on Windows, by @easyaspi314 +- cli : can `-c` check file generated by sfv +- build: `make DISPATCH=1` generates `xxhsum` and `libxxhash` with runtime vector detection (x86/x64 only) +- install: cygwin installation support +- doc : Cryptol specification of XXH32 and XXH64, by @weaversa + +v0.7.3 +- perf: improved speed for large inputs (~+20%) +- perf: improved latency for small inputs (~10%) +- perf: s390x Vectorial code, by @easyaspi314 +- cli: improved support for Unicode filenames on Windows, thanks to @easyaspi314 and @t-mat +- api: `xxhash.h` can now be included in any order, with and without `XXH_STATIC_LINKING_ONLY` and `XXH_INLINE_ALL` +- build: xxHash's implementation transferred into `xxhash.h`. No more need to have `xxhash.c` in the `/include` directory for `XXH_INLINE_ALL` to work +- install: created pkg-config file, by @bket +- install: VCpkg installation instructions, by @LilyWangL +- doc: Highly improved code documentation, by @easyaspi314 +- misc: New test tool in `/tests/collisions`: brute force collision tester for 64-bit hashes + +v0.7.2 +- Fixed collision ratio of `XXH128` for some specific input lengths, reported by @svpv +- Improved `VSX` and `NEON` variants, by @easyaspi314 +- Improved performance of scalar code path (`XXH_VECTOR=0`), by @easyaspi314 +- `xxhsum`: can generate 128-bit hashes with the `-H2` option (note: for experimental purposes only! `XXH128` is not yet frozen) +- `xxhsum`: option `-q` removes status notifications + +v0.7.1 +- Secret first: the algorithm computation can be altered by providing a "secret", which is any blob of bytes, of size >= `XXH3_SECRET_SIZE_MIN`. +- `seed` is still available, and acts as a secret generator +- updated `ARM NEON` variant by @easyaspi314 +- Streaming implementation is available +- Improve compatibility and performance with Visual Studio, with help from @aras-p +- Better integration when using `XXH_INLINE_ALL`: do not pollute host namespace, use its own macros, such as `XXH_ASSERT()`, `XXH_ALIGN`, etc. +- 128-bit variant provides helper functions for comparison of hashes. +- Better `clang` generation of `rotl` instruction, thanks to @easyaspi314 +- `XXH_REROLL` build macro to reduce binary size, by @easyaspi314 +- Improved `cmake` script, by @Mezozoysky +- Full benchmark program provided in `/tests/bench` diff --git a/External/xxHash/0.8.1/LICENSE b/External/xxHash/0.8.1/LICENSE new file mode 100644 index 000000000..6bc30a1bc --- /dev/null +++ b/External/xxHash/0.8.1/LICENSE @@ -0,0 +1,26 @@ +xxHash Library +Copyright (c) 2012-2020 Yann Collet +All rights reserved. + +BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/External/xxHash/0.8.1/README.md b/External/xxHash/0.8.1/README.md new file mode 100644 index 000000000..2406c8d24 --- /dev/null +++ b/External/xxHash/0.8.1/README.md @@ -0,0 +1,236 @@ + +xxHash - Extremely fast hash algorithm +====================================== + +xxHash is an Extremely fast Hash algorithm, running at RAM speed limits. +It successfully completes the [SMHasher](https://code.google.com/p/smhasher/wiki/SMHasher) test suite +which evaluates collision, dispersion and randomness qualities of hash functions. +Code is highly portable, and hashes are identical across all platforms (little / big endian). + +|Branch |Status | +|------------|---------| +|dev | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Adev+) | + + +Benchmarks +------------------------- + +The reference system uses an Intel i7-9700K cpu, and runs Ubuntu x64 20.04. +The [open source benchmark program] is compiled with `clang` v10.0 using `-O3` flag. + +| Hash Name | Width | Bandwidth (GB/s) | Small Data Velocity | Quality | Comment | +| --------- | ----- | ---------------- | ----- | --- | --- | +| __XXH3__ (SSE2) | 64 | 31.5 GB/s | 133.1 | 10 +| __XXH128__ (SSE2) | 128 | 29.6 GB/s | 118.1 | 10 +| _RAM sequential read_ | N/A | 28.0 GB/s | N/A | N/A | _for reference_ +| City64 | 64 | 22.0 GB/s | 76.6 | 10 +| T1ha2 | 64 | 22.0 GB/s | 99.0 | 9 | Slightly worse [collisions] +| City128 | 128 | 21.7 GB/s | 57.7 | 10 +| __XXH64__ | 64 | 19.4 GB/s | 71.0 | 10 +| SpookyHash | 64 | 19.3 GB/s | 53.2 | 10 +| Mum | 64 | 18.0 GB/s | 67.0 | 9 | Slightly worse [collisions] +| __XXH32__ | 32 | 9.7 GB/s | 71.9 | 10 +| City32 | 32 | 9.1 GB/s | 66.0 | 10 +| Murmur3 | 32 | 3.9 GB/s | 56.1 | 10 +| SipHash | 64 | 3.0 GB/s | 43.2 | 10 +| FNV64 | 64 | 1.2 GB/s | 62.7 | 5 | Poor avalanche properties +| Blake2 | 256 | 1.1 GB/s | 5.1 | 10 | Cryptographic +| SHA1 | 160 | 0.8 GB/s | 5.6 | 10 | Cryptographic but broken +| MD5 | 128 | 0.6 GB/s | 7.8 | 10 | Cryptographic but broken + +[open source benchmark program]: https://github.com/Cyan4973/xxHash/tree/release/tests/bench +[collisions]: https://github.com/Cyan4973/xxHash/wiki/Collision-ratio-comparison#collision-study + +note 1: Small data velocity is a _rough_ evaluation of algorithm's efficiency on small data. For more detailed analysis, please refer to next paragraph. + +note 2: some algorithms feature _faster than RAM_ speed. In which case, they can only reach their full speed when input data is already in CPU cache (L3 or better). Otherwise, they max out on RAM speed limit. + +### Small data + +Performance on large data is only one part of the picture. +Hashing is also very useful in constructions like hash tables and bloom filters. +In these use cases, it's frequent to hash a lot of small data (starting at a few bytes). +Algorithm's performance can be very different for such scenarios, since parts of the algorithm, +such as initialization or finalization, become fixed cost. +The impact of branch mis-prediction also becomes much more present. + +XXH3 has been designed for excellent performance on both long and small inputs, +which can be observed in the following graph: + +![XXH3, latency, random size](https://user-images.githubusercontent.com/750081/61976089-aedeab00-af9f-11e9-9239-e5375d6c080f.png) + +For a more detailed analysis, visit the wiki : +https://github.com/Cyan4973/xxHash/wiki/Performance-comparison#benchmarks-concentrating-on-small-data- + +Quality +------------------------- + +Speed is not the only property that matters. +Produced hash values must respect excellent dispersion and randomness properties, +so that any sub-section of it can be used to maximally spread out a table or index, +as well as reduce the amount of collisions to the minimal theoretical level, following the [birthday paradox]. + +`xxHash` has been tested with Austin Appleby's excellent SMHasher test suite, +and passes all tests, ensuring reasonable quality levels. +It also passes extended tests from [newer forks of SMHasher], featuring additional scenarios and conditions. + +Finally, xxHash provides its own [massive collision tester](https://github.com/Cyan4973/xxHash/tree/dev/tests/collisions), +able to generate and compare billions of hashes to test the limits of 64-bit hash algorithms. +On this front too, xxHash features good results, in line with the [birthday paradox]. +A more detailed analysis is documented [in the wiki](https://github.com/Cyan4973/xxHash/wiki/Collision-ratio-comparison). + +[birthday paradox]: https://en.wikipedia.org/wiki/Birthday_problem +[newer forks of SMHasher]: https://github.com/rurban/smhasher + + +### Build modifiers + +The following macros can be set at compilation time to modify libxxhash's behavior. They are generally disabled by default. + +- `XXH_INLINE_ALL`: Make all functions `inline`, with implementations being directly included within `xxhash.h`. + Inlining functions is beneficial for speed on small keys. + It's _extremely effective_ when key length is expressed as _a compile time constant_, + with performance improvements observed in the +200% range . + See [this article](https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html) for details. +- `XXH_PRIVATE_API`: same outcome as `XXH_INLINE_ALL`. Still available for legacy support. + The name underlines that `XXH_*` symbols will not be exported. +- `XXH_NAMESPACE`: Prefixes all symbols with the value of `XXH_NAMESPACE`. + This macro can only use compilable character set. + Useful to evade symbol naming collisions, + in case of multiple inclusions of xxHash's source code. + Client applications still use the regular function names, + as symbols are automatically translated through `xxhash.h`. +- `XXH_FORCE_MEMORY_ACCESS`: The default method `0` uses a portable `memcpy()` notation. + Method `1` uses a gcc-specific `packed` attribute, which can provide better performance for some targets. + Method `2` forces unaligned reads, which is not standards compliant, but might sometimes be the only way to extract better read performance. + Method `3` uses a byteshift operation, which is best for old compilers which don't inline `memcpy()` or big-endian systems without a byteswap instruction +- `XXH_FORCE_ALIGN_CHECK`: Use a faster direct read path when input is aligned. + This option can result in dramatic performance improvement when input to hash is aligned on 32 or 64-bit boundaries, + when running on architectures unable to load memory from unaligned addresses, or suffering a performance penalty from it. + It is (slightly) detrimental on platform with good unaligned memory access performance (same instruction for both aligned and unaligned accesses). + This option is automatically disabled on `x86`, `x64` and `aarch64`, and enabled on all other platforms. +- `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on linux will require `-mavx2` for AVX2, and `-mavx512f` for AVX512). +- `XXH_NO_PREFETCH` : disable prefetching. Some platforms or situations may perform better without prefetching. XXH3 only. +- `XXH_PREFETCH_DIST` : select prefetching distance. For close-to-metal adaptation to specific hardware platforms. XXH3 only. +- `XXH_NO_INLINE_HINTS`: By default, xxHash uses `__attribute__((always_inline))` and `__forceinline` to improve performance at the cost of code size. + Defining this macro to 1 will mark all internal functions as `static`, allowing the compiler to decide whether to inline a function or not. + This is very useful when optimizing for smallest binary size, + and is automatically defined when compiling with `-O0`, `-Os`, `-Oz`, or `-fno-inline` on GCC and Clang. + This may also increase performance depending on compiler and architecture. +- `XXH32_ENDJMP`: Switch multi-branch finalization stage of XXH32 by a single jump. + This is generally undesirable for performance, especially when hashing inputs of random sizes. + But depending on exact architecture and compiler, a jump might provide slightly better performance on small inputs. Disabled by default. +- `XXH_STATIC_LINKING_ONLY`: gives access to internal state declaration, required for static allocation. + Incompatible with dynamic linking, due to risks of ABI changes. +- `XXH_NO_XXH3` : removes symbols related to `XXH3` (both 64 & 128 bits) from generated binary. + Useful to reduce binary size, notably for applications which do not use `XXH3`. +- `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit types (XXH3 and XXH64). Only XXH32 will be compiled. + Useful for targets (architectures and compilers) without 64-bit support. +- `XXH_IMPORT`: MSVC specific: should only be defined for dynamic linking, as it prevents linkage errors. +- `XXH_CPU_LITTLE_ENDIAN`: By default, endianness is determined by a runtime test resolved at compile time. + If, for some reason, the compiler cannot simplify the runtime test, it can cost performance. + It's possible to skip auto-detection and simply state that the architecture is little-endian by setting this macro to 1. + Setting it to 0 states big-endian. +- `XXH_DEBUGLEVEL` : When set to any value >= 1, enables `assert()` statements. + This (slightly) slows down execution, but may help finding bugs during debugging sessions. + +When compiling the Command Line Interface `xxhsum` with `make`, the following environment variables can also be set : +- `DISPATCH=1` : use `xxh_x86dispatch.c`, to automatically select between `scalar`, `sse2`, `avx2` or `avx512` instruction set at runtime, depending on local host. This option is only valid for `x86`/`x64` systems. + + +### Building xxHash - Using vcpkg + +You can download and install xxHash using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install xxhash + +The xxHash port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + + +### Example + +The simplest example calls xxhash 64-bit variant as a one-shot function +generating a hash value from a single buffer, and invoked from a C/C++ program: + +```C +#include "xxhash.h" + + (...) + XXH64_hash_t hash = XXH64(buffer, size, seed); +} +``` + +Streaming variant is more involved, but makes it possible to provide data incrementally: + +```C +#include "stdlib.h" /* abort() */ +#include "xxhash.h" + + +XXH64_hash_t calcul_hash_streaming(FileHandler fh) +{ + /* create a hash state */ + XXH64_state_t* const state = XXH64_createState(); + if (state==NULL) abort(); + + size_t const bufferSize = SOME_SIZE; + void* const buffer = malloc(bufferSize); + if (buffer==NULL) abort(); + + /* Initialize state with selected seed */ + XXH64_hash_t const seed = 0; /* or any other value */ + if (XXH64_reset(state, seed) == XXH_ERROR) abort(); + + /* Feed the state with input data, any size, any number of times */ + (...) + while ( /* some data left */ ) { + size_t const length = get_more_data(buffer, bufferSize, fh); + if (XXH64_update(state, buffer, length) == XXH_ERROR) abort(); + (...) + } + (...) + + /* Produce the final hash value */ + XXH64_hash_t const hash = XXH64_digest(state); + + /* State could be re-used; but in this example, it is simply freed */ + free(buffer); + XXH64_freeState(state); + + return hash; +} +``` + + +### License + +The library files `xxhash.c` and `xxhash.h` are BSD licensed. +The utility `xxhsum` is GPL licensed. + + +### Other programming languages + +Beyond the C reference version, +xxHash is also available from many different programming languages, +thanks to great contributors. +They are [listed here](http://www.xxhash.com/#other-languages). + + +### Packaging status + +Many distributions bundle a package manager +which allows easy xxhash installation as both a `libxxhash` library +and `xxhsum` command line interface. + +[![Packaging status](https://repology.org/badge/vertical-allrepos/xxhash.svg)](https://repology.org/project/xxhash/versions) + + +### Special Thanks + +- Takayuki Matsuoka, aka @t-mat, for creating `xxhsum -c` and great support during early xxh releases +- Mathias Westerdahl, aka @JCash, for introducing the first version of `XXH64` +- Devin Hussey, aka @easyaspi314, for incredible low-level optimizations on `XXH3` and `XXH128` diff --git a/External/xxHash/0.8.1/libxxhash.pc.in b/External/xxHash/0.8.1/libxxhash.pc.in new file mode 100644 index 000000000..28c164485 --- /dev/null +++ b/External/xxHash/0.8.1/libxxhash.pc.in @@ -0,0 +1,15 @@ +# xxHash - Extremely fast hash algorithm +# Copyright (C) 2012-2020, Yann Collet, Facebook +# BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +prefix=@PREFIX@ +exec_prefix=@EXECPREFIX@ +includedir=${prefix}/@INCLUDEDIR@ +libdir=${exec_prefix}/@LIBDIR@ + +Name: xxhash +Description: extremely fast hash algorithm +URL: http://www.xxhash.com/ +Version: @VERSION@ +Libs: -L${libdir} -lxxhash +Cflags: -I${includedir} diff --git a/External/xxHash/0.8.1/xxh3.h b/External/xxHash/0.8.1/xxh3.h new file mode 100644 index 000000000..f7dc1959b --- /dev/null +++ b/External/xxHash/0.8.1/xxh3.h @@ -0,0 +1,55 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Development source file for `xxh3` + * Copyright (C) 2019-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/* + * Note: This file used to host the source code of XXH3_* variants. + * during the development period. + * The source code is now properly integrated within xxhash.h. + * + * xxh3.h is no longer useful, + * but it is still provided for compatibility with source code + * which used to include it directly. + * + * Programs are now highly discouraged to include xxh3.h. + * Include `xxhash.h` instead, which is the officially supported interface. + * + * In the future, xxh3.h will start to generate warnings, then errors, + * then it will be removed from source package and from include directory. + */ + +/* Simulate the same impact as including the old xxh3.h source file */ + +#define XXH_INLINE_ALL +#include "xxhash.h" diff --git a/External/xxHash/0.8.1/xxh_x86dispatch.c b/External/xxHash/0.8.1/xxh_x86dispatch.c new file mode 100644 index 000000000..399bad904 --- /dev/null +++ b/External/xxHash/0.8.1/xxh_x86dispatch.c @@ -0,0 +1,770 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (C) 2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + + +/*! + * @file xxh_x86dispatch.c + * + * Automatic dispatcher code for the @ref xxh3_family on x86-based targets. + * + * Optional add-on. + * + * **Compile this file with the default flags for your target.** Do not compile + * with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`, there will be + * an error. See @ref XXH_X86DISPATCH_ALLOW_AVX for details. + * + * @defgroup dispatch x86 Dispatcher + * @{ + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)) +# error "Dispatching is currently only supported on x86 and x86_64." +#endif + +/*! + * @def XXH_X86DISPATCH_ALLOW_AVX + * @brief Disables the AVX sanity check. + * + * Don't compile xxh_x86dispatch.c with options like `-mavx*`, `-march=native`, + * or `/arch:AVX*`. It is intended to be compiled for the minimum target, and + * it selectively enables SSE2, AVX2, and AVX512 when it is needed. + * + * Using this option _globally_ allows this feature, and therefore makes it + * undefined behavior to execute on any CPU without said feature. + * + * Even if the source code isn't directly using AVX intrinsics in a function, + * the compiler can still generate AVX code from autovectorization and by + * "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128). + * + * Use the same flags that you use to compile the rest of the program; this + * file will safely generate SSE2, AVX2, and AVX512 without these flags. + * + * Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check, and feel free to open + * an issue if there is a target in the future where AVX is a default feature. + */ +#ifdef XXH_DOXYGEN +# define XXH_X86DISPATCH_ALLOW_AVX +#endif + +#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX) +# error "Do not compile xxh_x86dispatch.c with AVX enabled! See the comment above." +#endif + +#ifdef __has_include +# define XXH_HAS_INCLUDE(header) __has_include(header) +#else +# define XXH_HAS_INCLUDE(header) 0 +#endif + +/*! + * @def XXH_DISPATCH_SCALAR + * @brief Enables/dispatching the scalar code path. + * + * If this is defined to 0, SSE2 support is assumed. This reduces code size + * when the scalar path is not needed. + * + * This is automatically defined to 0 when... + * - SSE2 support is enabled in the compiler + * - Targeting x86_64 + * - Targeting Android x86 + * - Targeting macOS + */ +#ifndef XXH_DISPATCH_SCALAR +# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \ + || defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \ + || defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */ +# define XXH_DISPATCH_SCALAR 0 /* disable */ +# else +# define XXH_DISPATCH_SCALAR 1 +# endif +#endif +/*! + * @def XXH_DISPATCH_AVX2 + * @brief Enables/disables dispatching for AVX2. + * + * This is automatically detected if it is not defined. + * - GCC 4.7 and later are known to support AVX2, but >4.9 is required for + * to get the AVX2 intrinsics and typedefs without -mavx -mavx2. + * - Visual Studio 2013 Update 2 and later are known to support AVX2. + * - The GCC/Clang internal header `` is detected. While this is + * not allowed to be included directly, it still appears in the builtin + * include path and is detectable with `__has_include`. + * + * @see XXH_AVX2 + */ +#ifndef XXH_DISPATCH_AVX2 +# if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \ + || (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \ + || (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \ + || XXH_HAS_INCLUDE() /* GCC/Clang internal header */ +# define XXH_DISPATCH_AVX2 1 /* enable dispatch towards AVX2 */ +# else +# define XXH_DISPATCH_AVX2 0 +# endif +#endif /* XXH_DISPATCH_AVX2 */ + +/*! + * @def XXH_DISPATCH_AVX512 + * @brief Enables/disables dispatching for AVX512. + * + * Automatically detected if one of the following conditions is met: + * - GCC 4.9 and later are known to support AVX512. + * - Visual Studio 2017 and later are known to support AVX2. + * - The GCC/Clang internal header `` is detected. While this + * is not allowed to be included directly, it still appears in the builtin + * include path and is detectable with `__has_include`. + * + * @see XXH_AVX512 + */ +#ifndef XXH_DISPATCH_AVX512 +# if (defined(__GNUC__) \ + && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \ + || (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \ + || XXH_HAS_INCLUDE() /* GCC/Clang internal header */ +# define XXH_DISPATCH_AVX512 1 /* enable dispatch towards AVX512 */ +# else +# define XXH_DISPATCH_AVX512 0 +# endif +#endif /* XXH_DISPATCH_AVX512 */ + +/*! + * @def XXH_TARGET_SSE2 + * @brief Allows a function to be compiled with SSE2 intrinsics. + * + * Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used + * even with `-mno-sse2`. + * + * @def XXH_TARGET_AVX2 + * @brief Like @ref XXH_TARGET_SSE2, but for AVX2. + * + * @def XXH_TARGET_AVX512 + * @brief Like @ref XXH_TARGET_SSE2, but for AVX512. + */ +#if defined(__GNUC__) +# include /* SSE2 */ +# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 +# include /* AVX2, AVX512F */ +# endif +# define XXH_TARGET_SSE2 __attribute__((__target__("sse2"))) +# define XXH_TARGET_AVX2 __attribute__((__target__("avx2"))) +# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f"))) +#elif defined(_MSC_VER) +# include +# define XXH_TARGET_SSE2 +# define XXH_TARGET_AVX2 +# define XXH_TARGET_AVX512 +#else +# error "Dispatching is currently not supported for your compiler." +#endif + +#ifdef XXH_DISPATCH_DEBUG +/* debug logging */ +# include +# define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); } +#else +# define XXH_debugPrint(str) ((void)0) +# undef NDEBUG /* avoid redefinition */ +# define NDEBUG +#endif +#include + +#define XXH_INLINE_ALL +#define XXH_X86DISPATCH +#include "xxhash.h" + +/* + * Support both AT&T and Intel dialects + * + * GCC doesn't convert AT&T syntax to Intel syntax, and will error out if + * compiled with -masm=intel. Instead, it supports dialect switching with + * curly braces: { AT&T syntax | Intel syntax } + * + * Clang's integrated assembler automatically converts AT&T syntax to Intel if + * needed, making the dialect switching useless (it isn't even supported). + * + * Note: Comments are written in the inline assembly itself. + */ +#ifdef __clang__ +# define XXH_I_ATT(intel, att) att "\n\t" +#else +# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t" +#endif + +/*! + * @internal + * @brief Runs CPUID. + * + * @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively. + * @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }` + */ +static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd) +{ +#if defined(_MSC_VER) + __cpuidex(abcd, eax, ecx); +#else + xxh_u32 ebx, edx; +# if defined(__i386__) && defined(__PIC__) + __asm__( + "# Call CPUID\n\t" + "#\n\t" + "# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t" + "# EBX, so we use EDI instead.\n\t" + XXH_I_ATT("mov edi, ebx", "movl %%ebx, %%edi") + XXH_I_ATT("cpuid", "cpuid" ) + XXH_I_ATT("xchg edi, ebx", "xchgl %%ebx, %%edi") + : "=D" (ebx), +# else + __asm__( + "# Call CPUID\n\t" + XXH_I_ATT("cpuid", "cpuid") + : "=b" (ebx), +# endif + "+a" (eax), "+c" (ecx), "=d" (edx)); + abcd[0] = eax; + abcd[1] = ebx; + abcd[2] = ecx; + abcd[3] = edx; +#endif +} + +/* + * Modified version of Intel's guide + * https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family + */ + +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 +/*! + * @internal + * @brief Runs `XGETBV`. + * + * While the CPU may support AVX2, the operating system might not properly save + * the full YMM/ZMM registers. + * + * xgetbv is used for detecting this: Any compliant operating system will define + * a set of flags in the xcr0 register indicating how it saves the AVX registers. + * + * You can manually disable this flag on Windows by running, as admin: + * + * bcdedit.exe /set xsavedisable 1 + * + * and rebooting. Run the same command with 0 to re-enable it. + */ +static xxh_u64 XXH_xgetbv(void) +{ +#if defined(_MSC_VER) + return _xgetbv(0); /* min VS2010 SP1 compiler is required */ +#else + xxh_u32 xcr0_lo, xcr0_hi; + __asm__( + "# Call XGETBV\n\t" + "#\n\t" + "# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t" + "# the XGETBV opcode, so we encode it by hand instead.\n\t" + "# See for details.\n\t" + ".byte 0x0f, 0x01, 0xd0\n\t" + : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0)); + return xcr0_lo | ((xxh_u64)xcr0_hi << 32); +#endif +} +#endif + +#define XXH_SSE2_CPUID_MASK (1 << 26) +#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27)) +#define XXH_AVX2_CPUID_MASK (1 << 5) +#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1)) +#define XXH_AVX512F_CPUID_MASK (1 << 16) +#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1)) + +/*! + * @internal + * @brief Returns the best XXH3 implementation. + * + * Runs various CPUID/XGETBV tests to try and determine the best implementation. + * + * @ret The best @ref XXH_VECTOR implementation. + * @see XXH_VECTOR_TYPES + */ +static int XXH_featureTest(void) +{ + xxh_u32 abcd[4]; + xxh_u32 max_leaves; + int best = XXH_SCALAR; +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 + xxh_u64 xgetbv_val; +#endif +#if defined(__GNUC__) && defined(__i386__) + xxh_u32 cpuid_supported; + __asm__( + "# For the sake of ruthless backwards compatibility, check if CPUID\n\t" + "# is supported in the EFLAGS on i386.\n\t" + "# This is not necessary on x86_64 - CPUID is mandatory.\n\t" + "# The ID flag (bit 21) in the EFLAGS register indicates support\n\t" + "# for the CPUID instruction. If a software procedure can set and\n\t" + "# clear this flag, the processor executing the procedure supports\n\t" + "# the CPUID instruction.\n\t" + "# \n\t" + "#\n\t" + "# Routine is from .\n\t" + + "# Save EFLAGS\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# Store EFLAGS\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# Invert the ID bit in stored EFLAGS\n\t" + XXH_I_ATT("xor dword ptr[esp], 0x200000", "xorl $0x200000, (%%esp)") + "# Load stored EFLAGS (with ID bit inverted)\n\t" + XXH_I_ATT("popfd", "popfl" ) + "# Store EFLAGS again (ID bit may or not be inverted)\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t" + XXH_I_ATT("pop eax", "popl %%eax" ) + "# eax = whichever bits were changed\n\t" + XXH_I_ATT("xor eax, dword ptr[esp]", "xorl (%%esp), %%eax" ) + "# Restore original EFLAGS\n\t" + XXH_I_ATT("popfd", "popfl" ) + "# eax = zero if ID bit can't be changed, else non-zero\n\t" + XXH_I_ATT("and eax, 0x200000", "andl $0x200000, %%eax" ) + : "=a" (cpuid_supported) :: "cc"); + + if (XXH_unlikely(!cpuid_supported)) { + XXH_debugPrint("CPUID support is not detected!"); + return best; + } + +#endif + /* Check how many CPUID pages we have */ + XXH_cpuid(0, 0, abcd); + max_leaves = abcd[0]; + + /* Shouldn't happen on hardware, but happens on some QEMU configs. */ + if (XXH_unlikely(max_leaves == 0)) { + XXH_debugPrint("Max CPUID leaves == 0!"); + return best; + } + + /* Check for SSE2, OSXSAVE and xgetbv */ + XXH_cpuid(1, 0, abcd); + + /* + * Test for SSE2. The check is redundant on x86_64, but it doesn't hurt. + */ + if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK)) + return best; + + XXH_debugPrint("SSE2 support detected."); + + best = XXH_SSE2; +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 + /* Make sure we have enough leaves */ + if (XXH_unlikely(max_leaves < 7)) + return best; + + /* Test for OSXSAVE and XGETBV */ + if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK) + return best; + + /* CPUID check for AVX features */ + XXH_cpuid(7, 0, abcd); + + xgetbv_val = XXH_xgetbv(); +#if XXH_DISPATCH_AVX2 + /* Validate that AVX2 is supported by the CPU */ + if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK) + return best; + + /* Validate that the OS supports YMM registers */ + if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) { + XXH_debugPrint("AVX2 supported by the CPU, but not the OS."); + return best; + } + + /* AVX2 supported */ + XXH_debugPrint("AVX2 support detected."); + best = XXH_AVX2; +#endif +#if XXH_DISPATCH_AVX512 + /* Check if AVX512F is supported by the CPU */ + if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) { + XXH_debugPrint("AVX512F not supported by CPU"); + return best; + } + + /* Validate that the OS supports ZMM registers */ + if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) { + XXH_debugPrint("AVX512F supported by the CPU, but not the OS."); + return best; + } + + /* AVX512F supported */ + XXH_debugPrint("AVX512F support detected."); + best = XXH_AVX512; +#endif +#endif + return best; +} + + +/* === Vector implementations === */ + +/*! + * @internal + * @brief Defines the various dispatch functions. + * + * TODO: Consolidate? + * + * @param suffix The suffix for the functions, e.g. sse2 or scalar + * @param target XXH_TARGET_* or empty. + */ +#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \ + \ +/* === XXH3, default variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_default_##suffix(const void* XXH_RESTRICT input, size_t len) \ +{ \ + return XXH3_hashLong_64b_internal( \ + input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH3, Seeded variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \ + XXH64_hash_t seed) \ +{ \ + return XXH3_hashLong_64b_withSeed_internal( \ + input, len, seed, XXH3_accumulate_512_##suffix, \ + XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \ + ); \ +} \ + \ +/* === XXH3, Secret variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \ + const void* secret, size_t secretLen) \ +{ \ + return XXH3_hashLong_64b_internal( \ + input, len, secret, secretLen, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH3 update variants === */ \ + \ +XXH_NO_INLINE target XXH_errorcode \ +XXH3_update_##suffix(XXH3_state_t* state, const void* input, size_t len) \ +{ \ + return XXH3_update(state, (const xxh_u8*)input, len, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \ +} \ + \ +/* === XXH128 default variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_default_##suffix(const void* XXH_RESTRICT input, size_t len) \ +{ \ + return XXH3_hashLong_128b_internal( \ + input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH128 Secret variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \ + const void* XXH_RESTRICT secret, size_t secretLen) \ +{ \ + return XXH3_hashLong_128b_internal( \ + input, len, (const xxh_u8*)secret, secretLen, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \ +} \ + \ +/* === XXH128 Seeded variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \ + XXH64_hash_t seed) \ +{ \ + return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix, \ + XXH3_initCustomSecret_##suffix); \ +} + +/* End XXH_DEFINE_DISPATCH_FUNCS */ + +#if XXH_DISPATCH_SCALAR +XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */) +#endif +XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2) +#if XXH_DISPATCH_AVX2 +XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2) +#endif +#if XXH_DISPATCH_AVX512 +XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512) +#endif +#undef XXH_DEFINE_DISPATCH_FUNCS + +/* ==== Dispatchers ==== */ + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(const void* XXH_RESTRICT, size_t); + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t); + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t); + +typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH3_state_t*, const void*, size_t); + +typedef struct { + XXH3_dispatchx86_hashLong64_default hashLong64_default; + XXH3_dispatchx86_hashLong64_withSeed hashLong64_seed; + XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret; + XXH3_dispatchx86_update update; +} XXH_dispatchFunctions_s; + +#define XXH_NB_DISPATCHES 4 + +/*! + * @internal + * @brief Table of dispatchers for @ref XXH3_64bits(). + * + * @pre The indices must match @ref XXH_VECTOR_TYPE. + */ +static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = { +#if XXH_DISPATCH_SCALAR + /* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar }, +#else + /* Scalar */ { NULL, NULL, NULL, NULL }, +#endif + /* SSE2 */ { XXHL64_default_sse2, XXHL64_seed_sse2, XXHL64_secret_sse2, XXH3_update_sse2 }, +#if XXH_DISPATCH_AVX2 + /* AVX2 */ { XXHL64_default_avx2, XXHL64_seed_avx2, XXHL64_secret_avx2, XXH3_update_avx2 }, +#else + /* AVX2 */ { NULL, NULL, NULL, NULL }, +#endif +#if XXH_DISPATCH_AVX512 + /* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 } +#else + /* AVX512 */ { NULL, NULL, NULL, NULL } +#endif +}; +/*! + * @internal + * @brief The selected dispatch table for @ref XXH3_64bits(). + */ +static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL }; + + +typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(const void* XXH_RESTRICT, size_t); + +typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t); + +typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t); + +typedef struct { + XXH3_dispatchx86_hashLong128_default hashLong128_default; + XXH3_dispatchx86_hashLong128_withSeed hashLong128_seed; + XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret; + XXH3_dispatchx86_update update; +} XXH_dispatch128Functions_s; + + +/*! + * @internal + * @brief Table of dispatchers for @ref XXH3_128bits(). + * + * @pre The indices must match @ref XXH_VECTOR_TYPE. + */ +static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = { +#if XXH_DISPATCH_SCALAR + /* Scalar */ { XXHL128_default_scalar, XXHL128_seed_scalar, XXHL128_secret_scalar, XXH3_update_scalar }, +#else + /* Scalar */ { NULL, NULL, NULL, NULL }, +#endif + /* SSE2 */ { XXHL128_default_sse2, XXHL128_seed_sse2, XXHL128_secret_sse2, XXH3_update_sse2 }, +#if XXH_DISPATCH_AVX2 + /* AVX2 */ { XXHL128_default_avx2, XXHL128_seed_avx2, XXHL128_secret_avx2, XXH3_update_avx2 }, +#else + /* AVX2 */ { NULL, NULL, NULL, NULL }, +#endif +#if XXH_DISPATCH_AVX512 + /* AVX512 */ { XXHL128_default_avx512, XXHL128_seed_avx512, XXHL128_secret_avx512, XXH3_update_avx512 } +#else + /* AVX512 */ { NULL, NULL, NULL, NULL } +#endif +}; + +/*! + * @internal + * @brief The selected dispatch table for @ref XXH3_64bits(). + */ +static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL }; + +/*! + * @internal + * @brief Runs a CPUID check and sets the correct dispatch tables. + */ +static void XXH_setDispatch(void) +{ + int vecID = XXH_featureTest(); + XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1); + assert(XXH_SCALAR <= vecID && vecID <= XXH_AVX512); +#if !XXH_DISPATCH_SCALAR + assert(vecID != XXH_SCALAR); +#endif +#if !XXH_DISPATCH_AVX512 + assert(vecID != XXH_AVX512); +#endif +#if !XXH_DISPATCH_AVX2 + assert(vecID != XXH_AVX2); +#endif + XXH_g_dispatch = XXH_kDispatch[vecID]; + XXH_g_dispatch128 = XXH_kDispatch128[vecID]; +} + + +/* ==== XXH3 public functions ==== */ + +static XXH64_hash_t +XXH3_hashLong_64b_defaultSecret_selection(const void* input, size_t len, + XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + if (XXH_g_dispatch.hashLong64_default == NULL) XXH_setDispatch(); + return XXH_g_dispatch.hashLong64_default(input, len); +} + +XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_defaultSecret_selection); +} + +static XXH64_hash_t +XXH3_hashLong_64b_withSeed_selection(const void* input, size_t len, + XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + if (XXH_g_dispatch.hashLong64_seed == NULL) XXH_setDispatch(); + return XXH_g_dispatch.hashLong64_seed(input, len, seed64); +} + +XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed_selection); +} + +static XXH64_hash_t +XXH3_hashLong_64b_withSecret_selection(const void* input, size_t len, + XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen) +{ + (void)seed64; + if (XXH_g_dispatch.hashLong64_secret == NULL) XXH_setDispatch(); + return XXH_g_dispatch.hashLong64_secret(input, len, secret, secretLen); +} + +XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_64b_withSecret_selection); +} + +XXH_errorcode +XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len) +{ + if (XXH_g_dispatch.update == NULL) XXH_setDispatch(); + return XXH_g_dispatch.update(state, (const xxh_u8*)input, len); +} + + +/* ==== XXH128 public functions ==== */ + +static XXH128_hash_t +XXH3_hashLong_128b_defaultSecret_selection(const void* input, size_t len, + XXH64_hash_t seed64, const void* secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + if (XXH_g_dispatch128.hashLong128_default == NULL) XXH_setDispatch(); + return XXH_g_dispatch128.hashLong128_default(input, len); +} + +XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_defaultSecret_selection); +} + +static XXH128_hash_t +XXH3_hashLong_128b_withSeed_selection(const void* input, size_t len, + XXH64_hash_t seed64, const void* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + if (XXH_g_dispatch128.hashLong128_seed == NULL) XXH_setDispatch(); + return XXH_g_dispatch128.hashLong128_seed(input, len, seed64); +} + +XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed_selection); +} + +static XXH128_hash_t +XXH3_hashLong_128b_withSecret_selection(const void* input, size_t len, + XXH64_hash_t seed64, const void* secret, size_t secretLen) +{ + (void)seed64; + if (XXH_g_dispatch128.hashLong128_secret == NULL) XXH_setDispatch(); + return XXH_g_dispatch128.hashLong128_secret(input, len, secret, secretLen); +} + +XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen) +{ + return XXH3_128bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_128b_withSecret_selection); +} + +XXH_errorcode +XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len) +{ + if (XXH_g_dispatch128.update == NULL) XXH_setDispatch(); + return XXH_g_dispatch128.update(state, (const xxh_u8*)input, len); +} + +#if defined (__cplusplus) +} +#endif +/*! @} */ diff --git a/External/xxHash/0.8.1/xxh_x86dispatch.h b/External/xxHash/0.8.1/xxh_x86dispatch.h new file mode 100644 index 000000000..6bc17bcbb --- /dev/null +++ b/External/xxHash/0.8.1/xxh_x86dispatch.h @@ -0,0 +1,86 @@ +/* + * xxHash - XXH3 Dispatcher for x86-based targets + * Copyright (C) 2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#ifndef XXH_X86DISPATCH_H_13563687684 +#define XXH_X86DISPATCH_H_13563687684 + +#include "xxhash.h" /* XXH64_hash_t, XXH3_state_t */ + +#if defined (__cplusplus) +extern "C" { +#endif + +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen); +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len); + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len); + +#if defined (__cplusplus) +} +#endif + + +/* automatic replacement of XXH3 functions. + * can be disabled by setting XXH_DISPATCH_DISABLE_REPLACE */ +#ifndef XXH_DISPATCH_DISABLE_REPLACE + +# undef XXH3_64bits +# define XXH3_64bits XXH3_64bits_dispatch +# undef XXH3_64bits_withSeed +# define XXH3_64bits_withSeed XXH3_64bits_withSeed_dispatch +# undef XXH3_64bits_withSecret +# define XXH3_64bits_withSecret XXH3_64bits_withSecret_dispatch +# undef XXH3_64bits_update +# define XXH3_64bits_update XXH3_64bits_update_dispatch + +# undef XXH128 +# define XXH128 XXH3_128bits_withSeed_dispatch +# define XXH3_128bits XXH3_128bits_dispatch +# undef XXH3_128bits +# define XXH3_128bits XXH3_128bits_dispatch +# undef XXH3_128bits_withSeed +# define XXH3_128bits_withSeed XXH3_128bits_withSeed_dispatch +# undef XXH3_128bits_withSecret +# define XXH3_128bits_withSecret XXH3_128bits_withSecret_dispatch +# undef XXH3_128bits_update +# define XXH3_128bits_update XXH3_128bits_update_dispatch + +#endif /* XXH_DISPATCH_DISABLE_REPLACE */ + +#endif /* XXH_X86DISPATCH_H_13563687684 */ diff --git a/External/xxHash/0.8.1/xxhash.c b/External/xxHash/0.8.1/xxhash.c new file mode 100644 index 000000000..0fae88c5d --- /dev/null +++ b/External/xxHash/0.8.1/xxhash.c @@ -0,0 +1,43 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/External/xxHash/0.8.1/xxhash.h b/External/xxHash/0.8.1/xxhash.h new file mode 100644 index 000000000..08ab79457 --- /dev/null +++ b/External/xxHash/0.8.1/xxhash.h @@ -0,0 +1,5580 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +/* TODO: update */ +/* Notice extracted from xxHash homepage: + +xxHash is an extremely fast hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MurmurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +Note: SMHasher's CRC32 implementation is not the fastest one. +Other speed-oriented implementations can be faster, +especially in combination with PCLMUL instruction: +https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + + + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 1 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return `XXH_VERSION_NUMBER` of the invoked library. + */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# else +# if ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +# endif +#endif + +/*! + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that @ref xxh3_family provides competitive speed + * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif + +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* +Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute +introduced in CPP17 and C23. +CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough +C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough +*/ +#if XXH_HAS_C_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_CPP_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((fallthrough)) +#else +# define XXH_FALLTHROUGH +#endif + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. + * + * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/* XXH3_64bits(): + * default 64-bit variant, using default secret and default seed of 0. + * It's the fastest variant. */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); + +/* + * XXH3_64bits_withSeed(): + * This variant generates a custom secret on the fly + * based on default secret altered using the `seed` value. + * While this operation is decently fast, note that it's not completely free. + * Note: seed==0 produces the same results as XXH3_64bits(). + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing "XXH3_generateSecret()" instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); + +/* + * XXH3_64bits_reset(): + * Initialize with default parameters. + * digest will be equivalent to `XXH3_64bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); +/* + * XXH3_64bits_reset_withSeed(): + * Generate a custom secret from `seed`, and store it into `statePtr`. + * digest will be equivalent to `XXH3_64bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +/* + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * XXH128_cmp(): + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); + + +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may be removed. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it may be removed. */ +}; /* typedef'd to XXH64_state_t */ + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; } + + +/* XXH128() : + * simple alias to pre-selected XXH3_128bits variant + */ +XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/* + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, + * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @secretSize + * into an already allocated buffer @secretBuffer. + * @secretSize must be >= XXH3_SECRET_SIZE_MIN + * + * The generated secret can then be used with any `*_withSecret()` variant. + * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, + * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so + * XXH3_generateSecret() can be employed to ensure proper quality. + * + * customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes. + * The resulting `secret` will nonetheless provide all required qualities. + * + * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize); + + +/* + * XXH3_generateSecret_fromSeed(): + * + * Generate the same secret as the _withSeed() variants. + * + * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily). + * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * This generator is notably useful in combination with `_withSecretandSeed()`, + * as a way to emulate a faster `_withSeed()` variant. + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed); + +/* + * *_withSecretandSeed() : + * These variants generate hash values using either + * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes) + * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, + * via its impact to the seed. + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed); + +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + + +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64 & arm64, + * which are platforms known to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ +# if !defined(__clang__) && \ +( \ + (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + ( \ + defined(__GNUC__) && ( \ + (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ + ( \ + defined(__mips__) && \ + (__mips <= 5 || __mips_isa_rev < 6) && \ + (!defined(__mips16) || defined(__mips_mips16e2)) \ + ) \ + ) \ + ) \ +) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ + || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# define XXH_ASSERT(c) ((void)0) +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# include +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; + return ((const xxh_unalign*)ptr)->u32; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 h32) +{ + h32 ^= h32 >> 15; + h32 *= XXH_PRIME32_2; + h32 ^= h32 >> 13; + h32 *= XXH_PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ +static xxh_u32 +XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + h32 += (*ptr++) * XXH_PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(h32); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 4: XXH_PROCESS4; + return XXH32_avalanche(h32); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 0: return XXH32_avalanche(h32); + } + XXH_ASSERT(0); + return h32; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +/*! + * @ingroup xxh32_family + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + state.v[1] = seed + XXH_PRIME32_2; + state.v[2] = seed + 0; + state.v[3] = seed - XXH_PRIME32_1; + /* do not write into reserved, planned to be removed in a future version */ + XXH_memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! + * @ingroup xxh32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer, but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; + return ((const xxh_unalign64*)ptr)->u64; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +static xxh_u64 XXH64_avalanche(xxh_u64 h64) +{ + h64 ^= h64 >> 33; + h64 *= XXH_PRIME64_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +static xxh_u64 +XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(h64); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (inputtotal_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#else +/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#if defined(__GNUC__) +# if defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# elif defined(__ARM_NEON__) || defined(__ARM_NEON) +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# endif +#elif defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment reqired for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM64) || defined(_M_ARM_ARMV7VE) /* msvc */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + + +#if XXH_VECTOR == XXH_NEON +/* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split into + * two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] + * | '---------. .--------' | + * | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 is + * completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so modifying + * D11 will modify the high half of Q5. This is similar to how modifying AH + * will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd lanes + * in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead of + * taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the + * lower and upper halves of the Q register to end up with the high and low + * halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions modifying two + * registers at once is not possible in GCC or Clang's IR, and they have to + * create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, many + * quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes to a + * D-form register zero the high bits, similar to how writes to W-form scalar + * registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent + * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one + * operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be this + * mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } + * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } + * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single VZIP.32. + */ + +/*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) + * { + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ +# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && defined(__GNUC__) \ + && !defined(__aarch64__) && !defined(__arm64__) && !defined(_M_ARM64) +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ + __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ + (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + } while (0) +# else +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + (outLo) = vmovn_u64 (in); \ + (outHi) = vshrn_n_u64 ((in), 32); \ + } while (0) +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +# if defined(__s390x__) +# include +# else +/* gcc's altivec.h can have the unwanted consequence to unconditionally + * #define bool, vector, and pixel keywords, + * with bad consequences for programs already using these keywords for other purposes. + * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, + * but it seems that, in some cases, it isn't. + * Force the build macro to be defined, so that keywords are not altered. + */ +# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) +# define __APPLE_ALTIVEC__ +# endif +# include +# endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# include +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if defined(__GNUC__) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif defined(_M_X64) || defined(_M_IA64) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= 0x165667919E3779F9ULL; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= 0x9FB21C651E98DF25ULL; + h64 ^= (h64 >> 35) + len ; + h64 *= 0x9FB21C651E98DF25ULL; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); + + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + int const nbRounds = (int)len / 16; + int i; + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + acc = XXH3_avalanche(acc); + XXH_ASSERT(nbRounds >= 8); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + /* last bytes */ + acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + return XXH3_avalanche(acc); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', + * this will warn "discards 'const' qualifier". */ + union { + const __m512i* cp; + void* p; + } remote_const_void; + remote_const_void.cp = src + i; + dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + uint64x2_t* const xacc = (uint64x2_t *) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* const xinput = (const uint8_t *) input; + uint8_t const* const xsecret = (const uint8_t *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { + /* data_vec = xinput[i]; */ + uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key; + uint32x2_t data_key_lo, data_key_hi; + /* xacc[i] += swap(data_vec); */ + uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); + uint64x2_t const swapped = vextq_u64(data64, data64, 1); + xacc[i] = vaddq_u64 (xacc[i], swapped); + /* data_key = data_vec ^ key_vec; */ + data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); + /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (data_key >> 32); + * data_key = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ + xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); + + } + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { uint64x2_t* xacc = (uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(uint64x2_t); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); + uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec)); + + /* xacc[i] *= XXH_PRIME32_1 */ + uint32x2_t data_key_lo, data_key_hi; + /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (xacc[i] >> 32); + * xacc[i] = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + { /* + * prod_hi = (data_key >> 32) * XXH_PRIME32_1; + * + * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will + * incorrectly "optimize" this: + * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); + * shifted = vshll_n_u32(tmp, 32); + * to this: + * tmp = "vmulq_u64"(a, b); // no such thing! + * shifted = vshlq_n_u64(tmp, 32); + * + * However, unlike SSE, Clang lacks a 64-bit multiply routine + * for NEON, and it scalarizes two 64-bit multiplies instead. + * + * vmull_u32 has the same timing as vmul_u32, and it avoids + * this bug completely. + * See https://bugs.llvm.org/show_bug.cgi?id=39967 + */ + uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); + /* xacc[i] = prod_hi << 32; */ + xacc[i] = vshlq_n_u64(prod_hi, 32); + /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); + } + } } +} + +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + unsigned long long* const xacc = (unsigned long long*) acc; + xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ + xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = vec_xl(0, xacc + 2 * i); + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + /* xacc[i] = acc_vec; */ + vec_xst(acc_vec, 0, xacc + 2 * i); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_u64x2* const xacc = (xxh_u64x2*) acc; + const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +/* scalar variants - universal */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xinput = (const xxh_u8*) input; /* no alignment restriction */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + size_t i; + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + for (i=0; i < XXH_ACC_NB; i++) { + xxh_u64 const data_val = XXH_readLE64(xinput + 8*i); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8); + xacc[i ^ 1] += data_val; /* swap adjacent lanes */ + xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + size_t i; + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + for (i=0; i < XXH_ACC_NB; i++) { + xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i); + xxh_u64 acc64 = xacc[i]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[i] = acc64; + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__clang__) && defined(__aarch64__) + /* + * UGLY HACK: + * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), long MOVK chains stall the + * integer pipelines: + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes Clang to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + /* + * Note: in debug mode, this overrides the asm optimization + * and Clang will emit MOVK chains again. + */ + XXH_ASSERT(kSecretPtr == XXH3_kSecret); + + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes Clang to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + + + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * XXH3_accumulate() + * Loops over XXH3_accumulate_512(). + * Assumption: nbStripes will not overflow the secret size + */ +XXH_FORCE_INLINE void +XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes, + XXH3_f_accumulate_512 f_acc512) +{ + size_t n; + for (n = 0; n < nbStripes; n++ ) { + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; + XXH_PREFETCH(in + XXH_PREFETCH_DIST); + f_acc512(acc, + in, + secret + n*XXH_SECRET_CONSUME_RATE); + } +} + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* input, size_t len, + XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ + +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/* Note : when XXH3_consumeStripes() is invoked, + * there must be a guarantee that at least one more byte must be consumed from input + * so that the function can blindly consume all stripes using the "normal" secret segment */ +XXH_FORCE_INLINE void +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ + XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); + if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { + /* need a scrambling operation */ + size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; + size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); + f_scramble(acc, secret + secretLimit); + XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); + *nbStripesSoFarPtr = nbStripesAfterBlock; + } else { + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); + *nbStripesSoFarPtr += nbStripes; + } +} + +#ifndef XXH3_STREAM_USE_STACK +# ifndef __clang__ /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + + /* large input to consume : ingest per full block */ + if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar); + /* join to current block's end */ + { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar; + XXH_ASSERT(nbStripes <= nbStripes); + XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512); + f_scramble(acc, secret + state->secretLimit); + state->nbStripesSoFar = 0; + input += nbStripesToEnd * XXH_STRIPE_LEN; + nbStripes -= nbStripesToEnd; + } + /* consume per entire blocks */ + while(nbStripes >= state->nbStripesPerBlock) { + XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + state->secretLimit); + input += state->nbStripesPerBlock * XXH_STRIPE_LEN; + nbStripes -= state->nbStripesPerBlock; + } + /* consume last partial block */ + XXH3_accumulate(acc, input, secret, nbStripes, f_acc512); + input += nbStripes * XXH_STRIPE_LEN; + XXH_ASSERT(input < bEnd); /* at least some bytes left */ + state->nbStripesSoFar = nbStripes; + /* buffer predecessor of last partial stripe */ + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN); + } else { + /* content to consume <= block size */ + /* Consume input by a multiple of internal buffer size */ + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; + do { + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + input += XXH3_INTERNALBUFFER_SIZE; + } while (inputbuffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + } + } + + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate_512, XXH3_scrambleAcc); + /* last stripe */ + XXH3_accumulate_512(acc, + state->buffer + state->bufferedSize - XXH_STRIPE_LEN, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } else { /* bufferedSize < XXH_STRIPE_LEN */ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + XXH3_accumulate_512(acc, + lastStripe, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + int const nbRounds = (int)len / 32; + int i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + for (i=0; i<4; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + (32 * i), + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + XXH_ASSERT(nbRounds >= 4); + for (i=4 ; i < nbRounds; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + 0ULL - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ + +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +static void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize) +{ + XXH_ASSERT(secretBuffer != NULL); + if (secretBuffer == NULL) return XXH_ERROR; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } + XXH_ASSERT(customSeed != NULL); + if (customSeed == NULL) return XXH_ERROR; + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n Date: Sun, 12 Mar 2023 15:06:36 +1030 Subject: [PATCH 007/129] xxHash3: Expose xxHash3 64 for future use (2-3x faster than regular xxhash 64) --- Code/Core/CoreTest/CoreTest.bff | 1 + Code/Core/CoreTest/Tests/TestHash.cpp | 25 ++++ Code/Core/Math/xxHash.h | 21 ++++ Code/Tools/FBuild/FBuild/FBuild.bff | 1 + Code/Tools/FBuild/FBuildTest/FBuildTest.bff | 1 + .../FBuild/FBuildWorker/FBuildWorker.bff | 1 + Code/fbuild.bff | 6 +- External/xxHash/xxHash.bff | 119 ++++++++++++++++++ 8 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 External/xxHash/xxHash.bff diff --git a/Code/Core/CoreTest/CoreTest.bff b/Code/Core/CoreTest/CoreTest.bff index ed49de3f2..1a177e7c2 100644 --- a/Code/Core/CoreTest/CoreTest.bff +++ b/Code/Core/CoreTest/CoreTest.bff @@ -52,6 +52,7 @@ 'TestFrameWork-Lib-$Platform$-$BuildConfigName$' 'Core-Lib-$Platform$-$BuildConfigName$' 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/$ProjectName$$ExeExtension$' #if __WINDOWS__ diff --git a/Code/Core/CoreTest/Tests/TestHash.cpp b/Code/Core/CoreTest/Tests/TestHash.cpp index 3041ae1f7..7d42c3c7b 100644 --- a/Code/Core/CoreTest/Tests/TestHash.cpp +++ b/Code/Core/CoreTest/Tests/TestHash.cpp @@ -104,6 +104,15 @@ void TestHash::CompareHashTimes_Large() const OUTPUT( "xxHash-64 : %2.3fs @ %6.3f GiB/s (hash: %016" PRIx64 ")\n", (double)time, (double)speed, crc ); } + // xxHash3_64 + { + const Timer t; + const uint64_t crc = xxHash3::Calc64( data.Get(), dataSize ); + const float time = t.GetElapsed(); + const float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; + OUTPUT( "xxHash3-64 : %2.3fs @ %6.3f GiB/s (hash: %016" PRIx64 ")\n", (double)time, (double)speed, crc ); + } + // CRC32 - 8x8 slicing { const Timer t; @@ -192,6 +201,22 @@ void TestHash::CompareHashTimes_Small() const OUTPUT( "xxHash-64 : %2.3fs @ %6.3f GiB/s (hash: %016" PRIx64 ")\n", (double)time, (double)speed, crc ); } + // xxHash3 - 64 + { + const Timer t; + uint64_t crc( 0 ); + for ( size_t j = 0; j < numIterations; ++j ) + { + for ( size_t i = 0; i < numStrings; ++i ) + { + crc += xxHash3::Calc64( strings[ i ].Get(), strings[ i ].GetLength() ); + } + } + const float time = t.GetElapsed(); + const float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; + OUTPUT( "xxHash3-64 : %2.3fs @ %6.3f GiB/s (hash: %016" PRIx64 ")\n", (double)time, (double)speed, crc ); + } + // CRC32 - 8x8 slicing { const Timer t; diff --git a/Code/Core/Math/xxHash.h b/Code/Core/Math/xxHash.h index 0f494edc2..1dea11d68 100644 --- a/Code/Core/Math/xxHash.h +++ b/Code/Core/Math/xxHash.h @@ -10,8 +10,12 @@ // avoid including xxhash header directly extern "C" { + // xxHash unsigned int XXH32( const void * input, size_t length, unsigned seed ); unsigned long long XXH64( const void * input, size_t length, unsigned long long seed ); + + // xxhash3 + unsigned long long xxHashLib_XXH3_64bits( const void * input, size_t length ); }; // xxHash @@ -28,6 +32,16 @@ class xxHash enum { XXHASH_SEED = 0x0 }; // arbitrarily chosen random seed }; +// xxHash3 +//------------------------------------------------------------------------------ +class xxHash3 +{ +public: + inline static uint64_t Calc64( const void * buffer, size_t len ); + + inline static uint64_t Calc64( const AString & string ) { return Calc64( string.Get(), string.GetLength() ); } +}; + // Calc32 //------------------------------------------------------------------------------ /*static*/ uint32_t xxHash::Calc32( const void * buffer, size_t len ) @@ -42,4 +56,11 @@ class xxHash return XXH64( buffer, len, XXHASH_SEED ); } +// Calc64 (xxHash3) +//------------------------------------------------------------------------------ +/*static*/ uint64_t xxHash3::Calc64( const void * buffer, size_t len ) +{ + return xxHashLib_XXH3_64bits( buffer, len ); +} + //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuild/FBuild.bff b/Code/Tools/FBuild/FBuild/FBuild.bff index d67691e85..8f4d31401 100644 --- a/Code/Tools/FBuild/FBuild/FBuild.bff +++ b/Code/Tools/FBuild/FBuild/FBuild.bff @@ -49,6 +49,7 @@ 'FBuildCore-Lib-$Platform$-$BuildConfigName$', 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } #if __LINUX__ .LinkerOutput = '$OutputBase$/$ProjectPath$/fbuild$ExeExtension$' // NOTE: lower case diff --git a/Code/Tools/FBuild/FBuildTest/FBuildTest.bff b/Code/Tools/FBuild/FBuildTest/FBuildTest.bff index 83edb1493..509e41c1a 100644 --- a/Code/Tools/FBuild/FBuildTest/FBuildTest.bff +++ b/Code/Tools/FBuild/FBuildTest/FBuildTest.bff @@ -54,6 +54,7 @@ 'TestFrameWork-Lib-$Platform$-$BuildConfigName$', 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/$ProjectName$$ExeExtension$' #if __WINDOWS__ diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff index d995390a1..37010caf3 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff @@ -79,6 +79,7 @@ 'OSUI-Lib-$Platform$-$BuildConfigName$' 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } #if __WINDOWS__ + 'FBuildWorker-Res-$Platform$-$BuildConfigName$' diff --git a/Code/fbuild.bff b/Code/fbuild.bff index a9d91365c..4b98093a2 100644 --- a/Code/fbuild.bff +++ b/Code/fbuild.bff @@ -394,6 +394,7 @@ Settings // External #include "..\External\LZ4\LZ4.bff" +#include "../External/xxHash/xxHash.bff" // Test Framework #include "TestFramework\TestFramework.bff" @@ -628,7 +629,7 @@ Alias( 'All+Tests' ) .Folder_External = [ .Path = 'External' - .Projects = { 'LZ4-proj', 'SDKs-proj' } + .Projects = { 'LZ4-proj', 'SDKs-proj', 'xxHash-proj' } ] .Folder_Test = [ @@ -668,7 +669,8 @@ Alias( 'All+Tests' ) 'FBuildWorker-xcodeproj' 'LZ4-xcodeproj' 'OSUI-xcodeproj' - 'TestFramework-xcodeproj' } + 'TestFramework-xcodeproj' + 'xxHash-xcodeproj' } .ProjectConfigs = {} ForEach( .BuildConfig in .BuildConfigs ) { diff --git a/External/xxHash/xxHash.bff b/External/xxHash/xxHash.bff new file mode 100644 index 000000000..f71cb9ce2 --- /dev/null +++ b/External/xxHash/xxHash.bff @@ -0,0 +1,119 @@ +// xxHash +//------------------------------------------------------------------------------ +.xxHashBasePath = '../External/xxHash/0.8.1/' +.xxHashIncludePaths = ' "-I$xxHashBasePath$"' +{ + .ProjectName = 'xxHash' + .ProjectPath = '$xxHashBasePath$' + + // Target/Compiler specific options + .xxHashOptions_x64 = [ + .xxHashCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + .xxHashOptions_x64Clang = [ + .xxHashCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + .xxHashOptions_x64Linux = [ + .xxHashCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + .xxHashOptions_x64ClangLinux = .xxHashOptions_x64Linux + .xxHashOptions_x64OSX = [ + .xxHashCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + .xxHashOptions_ARMOSX = [ + .xxHashCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + + // Library + //-------------------------------------------------------------------------- + .ProjectConfigs = {} + ForEach( .BuildConfig in .BuildConfigs ) + { + Using( .BuildConfig ) + + .OutputBase + '\$Platform$-$BuildConfigName$' + + Using( ."xxHashOptions_$Platform$" ) + + // Static Library + ObjectList( '$ProjectName$-Lib-$Platform$-$BuildConfigName$' ) + { + // Input - Only build specific files we use + .CompilerInputFiles = { + '$xxHashBasePath$\xxhash.c' + } + + // Options + .CompilerOptions = .CompilerOptionsC + + .xxHashIncludePaths + + .xxHashCompilerOptions + + // Place xxHash functions in a namespace to avoid collisions with lz4 + + ' "-DXXH_NAMESPACE=xxHashLib_"' + + // TODO:C Figure out why static_asserts don't compile properly + + ' "-DXXH_STATIC_ASSERT(x)="' + + #if __WINDOWS__ + // Remove flags that disable opimizations + - ' /Od' + - ' /RTC1' + + // Disable static analysis if enabled + // (we won't fix warnings in 3rd party code) + - ' --analyze' + - .StaticAnalysisOptions + #else + - ' -O0' + #endif + + // Disable warnings if using Clang. There are too many warnings in xxHash + // and they differ with every version of Clang + - ' -Wall' + - ' -Werror' + - ' -Wfatal-errors' + - ' -Wextra' + - ' -Wshadow' + - ' -Weverything' + + // Output + .CompilerOutputPath = '$OutputBase$/External/$ProjectName$/' + } + Alias( '$ProjectName$-$Platform$-$BuildConfigName$' ) { .Targets = '$ProjectName$-Lib-$Platform$-$BuildConfigName$' } + + #if __WINDOWS__ + .ProjectConfig = [ Using( .'Project_$Platform$_$BuildConfigName$' ) .Target = '$ProjectName$-$Platform$-$BuildConfigName$' ] + ^ProjectConfigs + .ProjectConfig + #endif + #if __OSX__ + .ProjectConfig = [ .Config = '$BuildConfigName$' .Target = '$ProjectName$-x64OSX-$BuildConfigName$' ] + ^ProjectConfigs + .ProjectConfig + #endif + } + + // Aliases + //-------------------------------------------------------------------------- + CreateCommonAliases( .ProjectName ) + + // Visual Studio Project Generation + //-------------------------------------------------------------------------- + #if __WINDOWS__ + .ExtraOptions = [ + .ProjectFiles = '../External/xxHash/xxHash.bff' + ] + CreateVCXProject_Lib( .ProjectName, .ProjectPath, .ProjectConfigs, .ExtraOptions ) + #endif + + // XCode Project Generation + //-------------------------------------------------------------------------- + #if __OSX__ + XCodeProject( '$ProjectName$-xcodeproj' ) + { + .ProjectOutput = '../tmp/XCode/Projects/0_External/$ProjectName$.xcodeproj/project.pbxproj' + .ProjectInputPaths = '$ProjectPath$/' + .ProjectBasePath = '$ProjectPath$/' + + .XCodeBuildWorkingDir = '../../../../Code/' + } + #endif +} From 678cadb89582234683d2fc41e617e53047d3cc37 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 18 Mar 2023 16:37:16 +1030 Subject: [PATCH 008/129] Threads: Replace most deprecated Thread API usage with newer/safer API - Also activate Thread test that was not enabled --- Code/Core/CoreTest/TestMain.cpp | 1 + Code/Core/CoreTest/Tests/TestAtomic.cpp | 11 +++-------- Code/Core/CoreTest/Tests/TestMutex.cpp | 11 +++-------- Code/Core/CoreTest/Tests/TestSemaphore.cpp | 8 +++----- .../CoreTest/Tests/TestSmallBlockAllocator.cpp | 18 +++++------------- Code/Core/CoreTest/Tests/TestThread.cpp | 3 +++ Code/Core/Process/Thread.cpp | 13 ++++++++++--- Code/Core/Process/Thread.h | 1 + .../FBuildCore/Helpers/BuildProfiler.cpp | 12 +++--------- .../FBuild/FBuildCore/Helpers/BuildProfiler.h | 2 +- .../FBuild/FBuildCore/Protocol/Client.cpp | 10 ++-------- Code/Tools/FBuild/FBuildCore/Protocol/Client.h | 2 +- .../FBuild/FBuildCore/Protocol/Server.cpp | 10 ++-------- Code/Tools/FBuild/FBuildCore/Protocol/Server.h | 2 +- .../FBuildTest/Tests/TestDistributed.cpp | 6 +++--- .../FBuild/FBuildTest/Tests/TestFastCancel.cpp | 6 +++--- .../FBuild/FBuildWorker/Worker/Worker.cpp | 10 ++++------ Code/Tools/FBuild/FBuildWorker/Worker/Worker.h | 2 -- 18 files changed, 49 insertions(+), 79 deletions(-) diff --git a/Code/Core/CoreTest/TestMain.cpp b/Code/Core/CoreTest/TestMain.cpp index e1b600f21..670a32d01 100644 --- a/Code/Core/CoreTest/TestMain.cpp +++ b/Code/Core/CoreTest/TestMain.cpp @@ -28,6 +28,7 @@ int main( int, char *[] ) REGISTER_TESTGROUP( TestSmallBlockAllocator ) REGISTER_TESTGROUP( TestSystemMutex ) REGISTER_TESTGROUP( TestTestTCPConnectionPool ) + REGISTER_TESTGROUP( TestThread ) REGISTER_TESTGROUP( TestTimer ) REGISTER_TESTGROUP( TestUnorderedMap ) diff --git a/Code/Core/CoreTest/Tests/TestAtomic.cpp b/Code/Core/CoreTest/Tests/TestAtomic.cpp index 666ea92f8..f88eb643f 100644 --- a/Code/Core/CoreTest/Tests/TestAtomic.cpp +++ b/Code/Core/CoreTest/Tests/TestAtomic.cpp @@ -51,19 +51,14 @@ class AtomicTestHelper TEST_ASSERT( m_Count2.Load() == initialValue ); // Spawn thread - Thread::ThreadHandle h = Thread::CreateThread( ThreadWrapper, - "AtomicTestHelper", - ( 64 * KILOBYTE ), - this ); + Thread t; + t.Start( ThreadWrapper, "AtomicTestHelper", this ); // Do works locally that mirrors the thread DoWork(); // Join thread - bool timedOut = false; - Thread::WaitForThread( h, 1000, timedOut ); - TEST_ASSERT( timedOut == false ); - Thread::CloseHandle( h ); + t.Join(); // Check expected results TEST_ASSERT( AtomicLoadRelaxed( &m_Count ) == expectedResult ); diff --git a/Code/Core/CoreTest/Tests/TestMutex.cpp b/Code/Core/CoreTest/Tests/TestMutex.cpp index ab9d9f4ed..f8d054374 100644 --- a/Code/Core/CoreTest/Tests/TestMutex.cpp +++ b/Code/Core/CoreTest/Tests/TestMutex.cpp @@ -88,10 +88,8 @@ void TestMutex::TestExclusivity() const data.m_Count = 0; data.m_BarrierCounter = 0; - Thread::ThreadHandle h = Thread::CreateThread( TestExclusivityThreadEntryFunction, - "TestExclusivity", - ( 64 * KILOBYTE ), - &data ); + Thread t; + t.Start( TestExclusivityThreadEntryFunction, "TestExclusivity", &data ); // arrive at barrier and wait AtomicInc( &data.m_BarrierCounter ); @@ -105,10 +103,7 @@ void TestMutex::TestExclusivity() const } // wait for other thread to complete - bool timedOut = false; - Thread::WaitForThread( h, 1000, timedOut ); - TEST_ASSERT( timedOut == false ); - Thread::CloseHandle( h ); + t.Join(); // ensure total is correct TEST_ASSERT( data.m_Count == 2000000 ); diff --git a/Code/Core/CoreTest/Tests/TestSemaphore.cpp b/Code/Core/CoreTest/Tests/TestSemaphore.cpp index 33175793b..c270ab50a 100644 --- a/Code/Core/CoreTest/Tests/TestSemaphore.cpp +++ b/Code/Core/CoreTest/Tests/TestSemaphore.cpp @@ -52,7 +52,8 @@ void TestSemaphore::WaitForSignal() const Semaphore s; // Create a thread which will signal the Semaphore - Thread::ThreadHandle h = Thread::CreateThread( WaitForSignal_Thread, "Test::WaitForSignal", ( 32 * KILOBYTE ), &s ); + Thread t; + t.Start( WaitForSignal_Thread, "Test::WaitForSignal", &s ); // Wait or the expected signal count for ( size_t i = 0; i < 100; ++i ) @@ -61,10 +62,7 @@ void TestSemaphore::WaitForSignal() const } // Cleanup thread - bool timedOut; - Thread::WaitForThread( h, 1000, timedOut ); - TEST_ASSERT( timedOut == false ); - Thread::CloseHandle( h ); + t.Join(); } // WaitForSignal_Thread diff --git a/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp b/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp index a1966c2c9..819af83fb 100644 --- a/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp +++ b/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp @@ -30,7 +30,7 @@ class TestSmallBlockAllocator : public TestGroup class ThreadInfo { public: - Thread::ThreadHandle m_ThreadHandle = INVALID_THREAD_HANDLE; + Thread m_Thread; Array< uint32_t > * m_AllocationSizes = nullptr; uint32_t m_RepeatCount = 0; float m_TimeTaken = 0.0f; @@ -101,17 +101,13 @@ void TestSmallBlockAllocator::MultiThreaded() const { info[ i ].m_AllocationSizes = & allocSizes; info[ i ].m_RepeatCount = repeatCount; - info[ i ].m_ThreadHandle = Thread::CreateThread( ThreadFunction_System, "SmallBlock", ( 64 * KILOBYTE ), (void*)&info[ i ] ); - TEST_ASSERT( info[ i ].m_ThreadHandle != INVALID_THREAD_HANDLE ); + info[ i ].m_Thread.Start( ThreadFunction_System, "SmallBlock", (void*)&info[ i ] ); } // Join the threads for ( size_t i = 0; i < numThreads; ++i ) { - bool timedOut; - Thread::WaitForThread( info[ i ].m_ThreadHandle, 500 * 1000, timedOut ); - Thread::CloseHandle( info[ i ].m_ThreadHandle ); - TEST_ASSERT( timedOut == false ); + info[ i ].m_Thread.Join(); time1 += info[ i ].m_TimeTaken; } } @@ -124,17 +120,13 @@ void TestSmallBlockAllocator::MultiThreaded() const { info[ i ].m_AllocationSizes = & allocSizes; info[ i ].m_RepeatCount = repeatCount; - info[ i ].m_ThreadHandle = Thread::CreateThread( ThreadFunction_SmallBlock, "SmallBlock", ( 64 * KILOBYTE ), (void*)&info[ i ] ); - TEST_ASSERT( info[ i ].m_ThreadHandle != INVALID_THREAD_HANDLE ); + info[ i ].m_Thread.Start( ThreadFunction_SmallBlock, "SmallBlock", (void*)&info[ i ] ); } // Join the threads for ( size_t i = 0; i < numThreads; ++i ) { - bool timedOut; - Thread::WaitForThread( info[ i ].m_ThreadHandle, 500 * 1000, timedOut ); - Thread::CloseHandle( info[ i ].m_ThreadHandle ); - TEST_ASSERT( timedOut == false ); + info[ i ].m_Thread.Join(); time2 += info[ i ].m_TimeTaken; } time2 /= numThreads; diff --git a/Code/Core/CoreTest/Tests/TestThread.cpp b/Code/Core/CoreTest/Tests/TestThread.cpp index ee14d7a3f..43451a3e0 100644 --- a/Code/Core/CoreTest/Tests/TestThread.cpp +++ b/Code/Core/CoreTest/Tests/TestThread.cpp @@ -41,6 +41,7 @@ void TestThread::Unused() const { // A thread object never used to create a thread Thread t; + TEST_ASSERT( t.IsRunning() == false ); } // StartAndJoin @@ -55,10 +56,12 @@ void TestThread::StartAndJoin() const t.Start( ThreadFunc, "StartAndJoin", reinterpret_cast( static_cast( userData ) ) ); + TEST_ASSERT( t.IsRunning() ); // Join and check result const uint32_t result = t.Join(); TEST_ASSERT( result == userData ); + TEST_ASSERT( t.IsRunning() == false ); } //------------------------------------------------------------------------------ diff --git a/Code/Core/Process/Thread.cpp b/Code/Core/Process/Thread.cpp index 165df23de..e5b134d6d 100644 --- a/Code/Core/Process/Thread.cpp +++ b/Code/Core/Process/Thread.cpp @@ -111,11 +111,11 @@ void Thread::Start( ThreadEntryFunction func, uint32_t stackSizeBytes ) { // Can only start if not already started - ASSERT( m_Handle == INVALID_THREAD_HANDLE ); + ASSERT( !IsRunning() ); // Start thread m_Handle = CreateThread( func, threadName, stackSizeBytes, userData ); - ASSERT( m_Handle != INVALID_THREAD_HANDLE ); + ASSERT( IsRunning() ); } // Join @@ -123,7 +123,7 @@ void Thread::Start( ThreadEntryFunction func, uint32_t Thread::Join() { // Must only join if running and not already joined - ASSERT( m_Handle != INVALID_THREAD_HANDLE ); + ASSERT( IsRunning() ); // Wait for thread and obtain return result // TODO:C Fix inconsistent return results when legacy API is removed @@ -136,6 +136,13 @@ uint32_t Thread::Join() return returnValue; } +// IsRunning +//------------------------------------------------------------------------------ +bool Thread::IsRunning() const +{ + return ( m_Handle != INVALID_THREAD_HANDLE ); +} + // GetCurrentThreadId //------------------------------------------------------------------------------ /*static*/ Thread::ThreadId Thread::GetCurrentThreadId() diff --git a/Code/Core/Process/Thread.h b/Code/Core/Process/Thread.h index dff648651..fd174a0c2 100644 --- a/Code/Core/Process/Thread.h +++ b/Code/Core/Process/Thread.h @@ -41,6 +41,7 @@ class Thread void * userData = nullptr, uint32_t stackSizeBytes = kDefaultStackSize ); uint32_t Join(); + bool IsRunning() const; // Thread Identification static ThreadId GetCurrentThreadId(); diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp index ddc7c3f24..49da73971 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp @@ -37,10 +37,8 @@ BuildProfiler::~BuildProfiler() = default; void BuildProfiler::StartMetricsGathering() { PROFILE_FUNCTION; - - ASSERT( m_Thread == INVALID_THREAD_HANDLE ); m_ThreadExit.Store( false ); - m_Thread = Thread::CreateThread( MetricsThreadWrapper, "BuildProfileMetrics" ); + m_Thread.Start( MetricsThreadWrapper, "BuildProfileMetrics" ); } // StopMetricsGathering @@ -48,13 +46,9 @@ void BuildProfiler::StartMetricsGathering() void BuildProfiler::StopMetricsGathering() { PROFILE_FUNCTION; - - ASSERT( m_Thread != INVALID_THREAD_HANDLE ); m_ThreadExit.Store( true ); m_ThreadSignalSemaphore.Signal(); - Thread::WaitForThread( m_Thread ); - Thread::CloseHandle( m_Thread ); - m_Thread = INVALID_THREAD_HANDLE; + m_Thread.Join(); } // RecordLocal @@ -113,7 +107,7 @@ void BuildProfiler::RecordRemote( uint32_t workerId, bool BuildProfiler::SaveJSON( const FBuildOptions & options, const char * fileName ) { // Thread must be stopped - ASSERT( m_Thread == INVALID_THREAD_HANDLE ); + ASSERT( m_Thread.IsRunning() == false ); // Record time taken to save (can't use regular macros since we're process those) const int64_t saveStart = Timer::GetNow(); diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.h b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.h index 35823cf4a..f2a460430 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.h +++ b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.h @@ -103,7 +103,7 @@ class BuildProfiler : public Singleton Mutex m_Mutex; Atomic m_ThreadExit{ false }; Semaphore m_ThreadSignalSemaphore; - Thread::ThreadHandle m_Thread = INVALID_THREAD_HANDLE; + Thread m_Thread; Array m_Events; Array m_Metrics; Array m_WorkerInfo; diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp index fae36c505..8a5a3e971 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp @@ -46,11 +46,7 @@ Client::Client( const Array< AString > & workerList, // allocate space for server states m_ServerList.SetSize( workerList.GetSize() ); - m_Thread = Thread::CreateThread( ThreadFuncStatic, - "Client", - ( 64 * KILOBYTE ), - this ); - ASSERT( m_Thread ); + m_Thread.Start( ThreadFuncStatic, "Client", this ); } // DESTRUCTOR @@ -62,11 +58,9 @@ Client::~Client() SetShuttingDown(); m_ShouldExit.Store( true ); - Thread::WaitForThread( m_Thread ); + m_Thread.Join(); ShutdownAllConnections(); - - Thread::CloseHandle( m_Thread ); } //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.h b/Code/Tools/FBuild/FBuildCore/Protocol/Client.h index 6a12213f9..8b3462475 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.h @@ -69,7 +69,7 @@ class Client : public TCPConnectionPool Array< AString > m_WorkerList; // workers to connect to Atomic m_ShouldExit; // signal from main thread bool m_DetailedLogging; - Thread::ThreadHandle m_Thread; // the thread to find and manage workers + Thread m_Thread; // the thread to find and manage workers // state Timer m_StatusUpdateTimer; diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index acf89fc38..23c93695f 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -34,11 +34,7 @@ Server::Server( uint32_t numThreadsInJobQueue ) { m_JobQueueRemote = FNEW( JobQueueRemote( numThreadsInJobQueue ? numThreadsInJobQueue : Env::GetNumProcessors() ) ); - m_Thread = Thread::CreateThread( ThreadFuncStatic, - "Server", - ( 64 * KILOBYTE ), - this ); - ASSERT( m_Thread ); + m_Thread.Start( ThreadFuncStatic, "Server", this ); } // DESTRUCTOR @@ -47,12 +43,10 @@ Server::~Server() { m_ShouldExit.Store( true ); JobQueueRemote::Get().WakeMainThread(); - Thread::WaitForThread( m_Thread ); + m_Thread.Join(); ShutdownAllConnections(); - Thread::CloseHandle( m_Thread ); - FDELETE m_JobQueueRemote; for ( ToolManifest * tool : m_Tools ) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h index 2b731e421..a4d654395 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h @@ -88,7 +88,7 @@ class Server : public TCPConnectionPool JobQueueRemote * m_JobQueueRemote; Atomic m_ShouldExit; // signal from main thread - Thread::ThreadHandle m_Thread; // the thread to manage workload + Thread m_Thread; // the thread to manage workload Mutex m_ClientListMutex; Array< ClientState * > m_ClientList; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp index a013272da..be1f266e2 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp @@ -426,14 +426,14 @@ void TestDistributed::ShutdownMemoryLeak() const } }; bool detectedDistributedJobs = false; - Thread::ThreadHandle h = Thread::CreateThread( Helper::AbortBuild, nullptr, 64 * KILOBYTE, &detectedDistributedJobs ); + Thread thread; + thread.Start( Helper::AbortBuild,"TestDistributed", &detectedDistributedJobs ); // Start build and check it was aborted TEST_ASSERT( fBuild.Build( "ShutdownMemoryLeak" ) == false ); TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Error: BUILD FAILED: ShutdownMemoryLeak" ) ); - Thread::WaitForThread( h ); - Thread::CloseHandle( h ); + thread.Join(); TEST_ASSERT( detectedDistributedJobs ); } diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp index 4ed69ab8f..3b7936ec7 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp @@ -84,14 +84,14 @@ void TestFastCancel::Cancel() const SystemMutex * mutexes[] = { &mutex1, &mutex2, &mutex3, &mutex4 }; // Create thread that will abort build once all processes are spawned - Thread::ThreadHandle h = Thread::CreateThread( CancelHelperThread ); + Thread thread; + thread.Start( CancelHelperThread ); // Start build and check it was aborted TEST_ASSERT( fBuild.Build( "Cancel" ) == false ); TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Error: BUILD FAILED: Cancel" ) ); - Thread::WaitForThread( h ); - Thread::CloseHandle( h ); + thread.Join(); // Ensure that processes were killed for ( SystemMutex * mutex : mutexes ) diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp index 5ffab8afd..a90f561ed 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp @@ -28,6 +28,7 @@ #include "Core/FileIO/FileIO.h" #include "Core/Network/NetworkStartupHelper.h" #include "Core/Process/Process.h" +#include "Core/Process/Thread.h" #include "Core/Profile/Profile.h" #include "Core/Strings/AStackString.h" #include "Core/Tracing/Tracing.h" @@ -132,11 +133,8 @@ int32_t Worker::Work() } // spawn work thread - m_WorkThread = Thread::CreateThread( &WorkThreadWrapper, - "WorkerThread", - ( 256 * KILOBYTE ), - this ); - ASSERT( m_WorkThread != INVALID_THREAD_HANDLE ); + Thread workThread; + workThread.Start( &WorkThreadWrapper, "WorkerThread", this, ( 256 * KILOBYTE ) ); // Run the UI message loop if we're not in console mode if ( m_MainWindow ) @@ -145,7 +143,7 @@ int32_t Worker::Work() } // Join work thread and get exit code - return Thread::WaitForThread( m_WorkThread ); + return static_cast( workThread.Join() ); } // WorkThreadWrapper diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h index aa4c1bff6..076817b45 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h @@ -13,7 +13,6 @@ #include "Core/Containers/Singleton.h" #include "Core/Env/MSVCStaticAnalysis.h" #include "Core/FileIO/FileStream.h" -#include "Core/Process/Thread.h" // Forward Declarations //------------------------------------------------------------------------------ @@ -72,7 +71,6 @@ class Worker : public Singleton int32_t m_LastMemoryCheckResult; // -1 : No check done yet. 0=Not enough memory right now. 1=OK for now. #endif mutable AString m_LastStatusMessage; - Thread::ThreadHandle m_WorkThread; }; //------------------------------------------------------------------------------ From 1e881ed1ea45691ff414d47abe23f5aae87e7ae2 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 19 Mar 2023 10:51:44 +1030 Subject: [PATCH 009/129] Core: Remove most of deprecated Thread API - use new API where possible - trim legacy API to only what remains in use - make Thread internally directly use what it needs instead of forwarding to the deprecated API --- .../CoreTest/Tests/TestTCPConnectionPool.cpp | 9 +- Code/Core/Network/Network.cpp | 15 +- Code/Core/Network/TCPConnectionPool.cpp | 20 +- Code/Core/Process/Thread.cpp | 235 ++++++++---------- Code/Core/Process/Thread.h | 13 +- .../FBuildCore/WorkerPool/WorkerThread.cpp | 9 +- .../FBuildCore/WorkerPool/WorkerThread.h | 2 + 7 files changed, 119 insertions(+), 184 deletions(-) diff --git a/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp b/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp index bafce54f7..0a4d09d7d 100644 --- a/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp +++ b/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp @@ -288,10 +288,8 @@ void TestTestTCPConnectionPool::TestConnectionStuckDuringSend() const TEST_ASSERT( ci ); // start a thread to flood the slow server - Thread::ThreadHandle h = Thread::CreateThread( TestConnectionStuckDuringSend_ThreadFunc, - "Sender", - ( 64 * KILOBYTE ), - (void *)ci ); + Thread thread; + thread.Start( TestConnectionStuckDuringSend_ThreadFunc, "Sender", (void *)ci ); // let thread send enough data to become blocked in Send Thread::Sleep( 100 ); @@ -301,8 +299,7 @@ void TestTestTCPConnectionPool::TestConnectionStuckDuringSend() const // wait for client thread to exit, with timeout bool timedOut( false ); - Thread::WaitForThread( h, 1000, timedOut ); - Thread::CloseHandle( h ); + thread.JoinWithTimeout( 1000, timedOut ); // TODO:B Remove use of unsafe API // if timeout was hit, things were stuck TEST_ASSERT( timedOut == false ); diff --git a/Code/Core/Network/Network.cpp b/Code/Core/Network/Network.cpp index 5be4d3e73..87913b648 100644 --- a/Code/Core/Network/Network.cpp +++ b/Code/Core/Network/Network.cpp @@ -173,19 +173,17 @@ data.safeToFree = false; // will be marked by other thread // Create thread to perform resolution - Thread::ThreadHandle handle = Thread::CreateThread( NameResolutionThreadFunc, - "NameResolution", - ( 32 * KILOBYTE ), - &data ); + Thread thread; + thread.Start( NameResolutionThreadFunc, "NameResolution", &data, ( 32 * KILOBYTE ) ); // wait for name resolution with timeout bool timedOut( false ); - int returnCode( 0 ); + uint32_t returnCode( 0 ); uint32_t remainingTimeMS( timeoutMS ); const uint32_t sleepInterval( 100 ); // Check exit condition periodically - TODO:C would be better to use an event for ( ;; ) { - returnCode = Thread::WaitForThread( handle, sleepInterval, timedOut ); + returnCode = thread.JoinWithTimeout( sleepInterval, timedOut ); // TODO:B Remove use of this unsafe API // Are we shutting down? if ( NetworkStartupHelper::IsShuttingDown() ) @@ -210,9 +208,8 @@ } if ( timedOut ) { - Thread::DetachThread( handle ); + thread.Detach(); // TODO:B Remove use of this unsafe API } - Thread::CloseHandle( handle ); // handle race where timeout occurred before thread marked data as // safe to delete (this could happen if system was under load and timeout was very small) @@ -227,7 +224,7 @@ } // return result of resolution (could also have failed) - return (uint32_t)returnCode; + return returnCode; } // NameResolutionThreadFunc diff --git a/Code/Core/Network/TCPConnectionPool.cpp b/Code/Core/Network/TCPConnectionPool.cpp index 04fb0dd79..2e9829c00 100644 --- a/Code/Core/Network/TCPConnectionPool.cpp +++ b/Code/Core/Network/TCPConnectionPool.cpp @@ -920,13 +920,9 @@ void TCPConnectionPool::CreateListenThread( TCPSocket socket, uint32_t host, uin m_ListenConnection->m_ThreadQuitNotification.Store( false ); // Spawn thread to handle socket - Thread::ThreadHandle h = Thread::CreateThread( &ListenThreadWrapperFunction, - "TCPListen", - ( 32 * KILOBYTE ), - m_ListenConnection ); // user data argument - ASSERT( h != INVALID_THREAD_HANDLE ); - Thread::DetachThread( h ); - Thread::CloseHandle( h ); // we don't need this anymore + Thread thread; + thread.Start( &ListenThreadWrapperFunction, "TCPListen", m_ListenConnection, ( 32 * KILOBYTE ) ); + thread.Detach(); // TODO: Remove use of this unsafe API } // ThreadWrapperFunction @@ -1043,13 +1039,9 @@ ConnectionInfo * TCPConnectionPool::CreateConnectionThread( TCPSocket socket, ui #endif // Spawn thread to handle socket - Thread::ThreadHandle h = Thread::CreateThread( &ConnectionThreadWrapperFunction, - "TCPConnection", - ( 64 * KILOBYTE ), - ci ); // user data argument - ASSERT( h != INVALID_THREAD_HANDLE ); - Thread::DetachThread( h ); - Thread::CloseHandle( h ); // we don't need this anymore + Thread thread; + thread.Start( &ConnectionThreadWrapperFunction, "TCPConnection", ci ); + thread.Detach(); // TODO:B Remove use of this unsafe API m_Connections.Append( ci ); diff --git a/Code/Core/Process/Thread.cpp b/Code/Core/Process/Thread.cpp index e5b134d6d..c35da8425 100644 --- a/Code/Core/Process/Thread.cpp +++ b/Code/Core/Process/Thread.cpp @@ -113,8 +113,44 @@ void Thread::Start( ThreadEntryFunction func, // Can only start if not already started ASSERT( !IsRunning() ); - // Start thread - m_Handle = CreateThread( func, threadName, stackSizeBytes, userData ); + // Create structure to pass to thread + ThreadStartInfo & info = *FNEW( ThreadStartInfo( func, userData, threadName ) ); + MemoryBarrier(); + + // Create thread + #if defined( __WINDOWS__ ) + HANDLE h = ::CreateThread( nullptr, // LPSECURITY_ATTRIBUTES lpThreadAttributes + stackSizeBytes, // SIZE_T dwStackSize + (LPTHREAD_START_ROUTINE)ThreadStartInfo::ThreadStartFunction, // LPTHREAD_START_ROUTINE lpStartAddress + &info, // LPVOID lpParameter + 0, // DWORD dwCreationFlags + nullptr // LPDWORD lpThreadId + ); + ASSERT( h != nullptr ); + #elif defined( __LINUX__ ) || defined( __APPLE__ ) + #if __has_feature( address_sanitizer ) || defined( __SANITIZE_ADDRESS__ ) + // AddressSanitizer instruments objects created on the stack by inserting redzones around them. + // This greatly increases the amount of stack space used by the program. + // To account for that double the requested stack size for the thread. + stackSizeBytes *= 2; + #endif + // Necessary on Aarch64, where it's 131072 in my tests. Sometimes we ask for 65536. + if ( stackSizeBytes < PTHREAD_STACK_MIN ) + { + stackSizeBytes = PTHREAD_STACK_MIN; + } + pthread_t h( 0 ); + pthread_attr_t threadAttr; + VERIFY( pthread_attr_init( &threadAttr ) == 0 ); + VERIFY( pthread_attr_setstacksize( &threadAttr, stackSizeBytes ) == 0 ); + VERIFY( pthread_attr_setdetachstate( &threadAttr, PTHREAD_CREATE_JOINABLE ) == 0 ); + VERIFY( pthread_create( &h, &threadAttr, ThreadStartInfo::ThreadStartFunction, &info ) == 0 ); + ASSERT( h != (pthread_t)0 ); + #else + #error Unknown platform + #endif + m_Handle = (Thread::ThreadHandle)h; + ASSERT( IsRunning() ); } @@ -126,14 +162,29 @@ uint32_t Thread::Join() ASSERT( IsRunning() ); // Wait for thread and obtain return result - // TODO:C Fix inconsistent return results when legacy API is removed - const uint32_t returnValue = static_cast( WaitForThread( m_Handle ) ); - - // Clean up the handle - CloseHandle( m_Handle ); - m_Handle = INVALID_THREAD_HANDLE; - - return returnValue; + #if defined( __WINDOWS__ ) + // Wait for thread to finish + VERIFY( WaitForSingleObject( m_Handle, INFINITE ) == WAIT_OBJECT_0 ); + + // Ge teturn code + DWORD ret = 0; + VERIFY( ::GetExitCodeThread( m_Handle, (LPDWORD)&ret ) ); + VERIFY( ::CloseHandle( m_Handle ) ); + m_Handle = INVALID_THREAD_HANDLE; + return ret; + #elif defined( __APPLE__ ) || defined( __LINUX__ ) + void * ret; + if ( pthread_join( (pthread_t)m_Handle, &ret ) == 0 ) + { + m_Handle = INVALID_THREAD_HANDLE; + return static_cast( (size_t)ret ); + } + ASSERT( false ); // usage failure + m_Handle = INVALID_THREAD_HANDLE; + return 0; + #else + #error Unknown platform + #endif } // IsRunning @@ -171,121 +222,60 @@ bool Thread::IsRunning() const #endif } -// CreateThread +// Detach //------------------------------------------------------------------------------ -/*static*/ Thread::ThreadHandle Thread::CreateThread( ThreadEntryFunction entryFunc, - const char * threadName, - uint32_t stackSize, - void * userData ) +void Thread::Detach() { - ThreadStartInfo & info = *FNEW( ThreadStartInfo( entryFunc, userData, threadName ) ); - MemoryBarrier(); - - #if defined( __WINDOWS__ ) - HANDLE h = ::CreateThread( nullptr, // LPSECURITY_ATTRIBUTES lpThreadAttributes - stackSize, // SIZE_T dwStackSize - (LPTHREAD_START_ROUTINE)ThreadStartInfo::ThreadStartFunction, // LPTHREAD_START_ROUTINE lpStartAddress - &info, // LPVOID lpParameter - 0, // DWORD dwCreationFlags - nullptr // LPDWORD lpThreadId - ); - ASSERT( h != nullptr ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) - #if __has_feature( address_sanitizer ) || defined( __SANITIZE_ADDRESS__ ) - // AddressSanitizer instruments objects created on the stack by inserting redzones around them. - // This greatly increases the amount of stack space used by the program. - // To account for that double the requested stack size for the thread. - stackSize *= 2; - #endif - // Necessary on Aarch64, where it's 131072 in my tests. Sometimes we ask for 65536. - if ( stackSize < PTHREAD_STACK_MIN ) - { - stackSize = PTHREAD_STACK_MIN; - } - pthread_t h( 0 ); - pthread_attr_t threadAttr; - VERIFY( pthread_attr_init( &threadAttr ) == 0 ); - VERIFY( pthread_attr_setstacksize( &threadAttr, stackSize ) == 0 ); - VERIFY( pthread_attr_setdetachstate( &threadAttr, PTHREAD_CREATE_JOINABLE ) == 0 ); - VERIFY( pthread_create( &h, &threadAttr, ThreadStartInfo::ThreadStartFunction, &info ) == 0 ); - ASSERT( h != (pthread_t)0 ); - #else - #error Unknown platform - #endif + // TODO:B Remove this unsafe function - return (Thread::ThreadHandle)h; -} + // Must only detach if running and not already joined or detached + ASSERT( IsRunning() ); -// WaitForThread -//------------------------------------------------------------------------------ -/*static*/ int32_t Thread::WaitForThread( ThreadHandle handle ) -{ #if defined( __WINDOWS__ ) - bool timedOut = true; // default is true to catch cases when timedOut wasn't set by WaitForThread() - const int32_t ret = WaitForThread( handle, INFINITE, timedOut ); - - if ( timedOut ) - { - // something is wrong - we were waiting an INFINITE time - ASSERT( false ); - return 0; - } - - return ret; + VERIFY( ::CloseHandle( m_Handle ) ); #elif defined( __APPLE__ ) || defined( __LINUX__ ) - void * ret; - if ( pthread_join( (pthread_t)handle, &ret ) == 0 ) - { - return (int)( (size_t)ret ); - } - ASSERT( false ); // usage failure - return 0; + VERIFY( pthread_detach( (pthread_t)m_Handle ) == 0 ); #else #error Unknown platform #endif + + m_Handle = INVALID_THREAD_HANDLE; } -// WaitForThread +// JoinWithTimeout //------------------------------------------------------------------------------ -/*static*/ int32_t Thread::WaitForThread( ThreadHandle handle, uint32_t timeoutMS, bool & timedOut ) +uint32_t Thread::JoinWithTimeout( uint32_t timeoutMS, bool & outTimedOut ) { + // TODO:B Remove this unsafe function + #if defined( __WINDOWS__ ) - // wait for thread to finish - const DWORD waitResult = WaitForSingleObject( handle, timeoutMS ); + // Wait for thread to finish + const DWORD waitResult = WaitForSingleObject( m_Handle, timeoutMS ); - // timed out ? + // Timed out? if ( waitResult == WAIT_TIMEOUT ) { - timedOut = true; + outTimedOut = true; return 0; } - if ( waitResult != WAIT_OBJECT_0 ) - { - // something is wrong - invalid handle? - ASSERT( false ); - timedOut = false; - return 0; - } - - // get actual return code - DWORD ret( 0 ); - if ( ::GetExitCodeThread( handle, (LPDWORD)&ret ) ) - { - timedOut = false; - return (int32_t)ret; - } - ASSERT( false ); // invalid thread handle - timedOut = false; - return -1; + VERIFY( waitResult == WAIT_OBJECT_0 ); // Invalid handle? + outTimedOut = false; + + // Get return code + DWORD ret = 0; + VERIFY( ::GetExitCodeThread( m_Handle, (LPDWORD)&ret ) ); + VERIFY( ::CloseHandle( m_Handle ) ); + m_Handle = INVALID_THREAD_HANDLE; + return ret; #elif __has_feature( thread_sanitizer ) || defined( __SANITIZE_THREAD__ ) // ThreadSanitizer doesn't support pthread_timedjoin_np yet (as of February 2018) - timedOut = false; + outTimedOut = false; (void)timeoutMS; - return WaitForThread( handle ); + return Join(); #elif defined( __APPLE__ ) - timedOut = false; + outTimedOut = false; (void)timeoutMS; // TODO:MAC Implement timeout support - return WaitForThread( handle ); + return Join(); #elif defined( __LINUX__ ) // timeout is specified in absolute time // - get current time @@ -305,47 +295,16 @@ bool Thread::IsRunning() const // join thread with timeout void * ret; - int retVal = pthread_timedjoin_np( (pthread_t)handle, &ret, &abstime ); + int retVal = pthread_timedjoin_np( (pthread_t)m_Handle, &ret, &abstime ); if ( ( retVal == EBUSY ) || ( retVal == ETIMEDOUT ) ) { - timedOut = true; + outTimedOut = true; return 0; } - if ( retVal == 0 ) - { - timedOut = false; - return (int)( (size_t)ret ); - } - - ASSERT( false ); // a non-timeout error indicates usage failure - timedOut = false; - return 0; - #else - #error Unknown platform - #endif -} - -// DetachThread -//------------------------------------------------------------------------------ -/*static*/ void Thread::DetachThread( ThreadHandle handle ) -{ - #if defined( __WINDOWS__ ) - (void)handle; // Nothing to do - #elif defined( __APPLE__ ) || defined( __LINUX__ ) - VERIFY( pthread_detach( (pthread_t)handle ) == 0 ); - #else - #error Unknown platform - #endif -} - -// CloseHandle -//------------------------------------------------------------------------------ -/*static*/ void Thread::CloseHandle( ThreadHandle h ) -{ - #if defined( __WINDOWS__ ) - ::CloseHandle( h ); - #elif defined( __APPLE__ ) || defined( __LINUX__ ) - (void)h; // Nothing to do + VERIFY( retVal == 0 ); + m_Handle = INVALID_THREAD_HANDLE; + outTimedOut = false; + return static_cast( (size_t)ret ); #else #error Unknown platform #endif diff --git a/Code/Core/Process/Thread.h b/Code/Core/Process/Thread.h index fd174a0c2..c62edfe43 100644 --- a/Code/Core/Process/Thread.h +++ b/Code/Core/Process/Thread.h @@ -52,16 +52,9 @@ class Thread // Sleeps static void Sleep( uint32_t ms ); - // Thread lifetime (Legacy API) - static ThreadHandle CreateThread( ThreadEntryFunction entryFunc, - const char * threadName = nullptr, - uint32_t stackSize = kDefaultStackSize, - void * userData = nullptr - ); - static int32_t WaitForThread( ThreadHandle handle ); - static int32_t WaitForThread( ThreadHandle handle, uint32_t timeoutMS, bool & timedOut ); - static void DetachThread( ThreadHandle handle ); - static void CloseHandle( ThreadHandle h ); + // Legacy Functions - TODO:B Remove these unsafe functions + void Detach(); // TODO:B Remove this unsafe function + uint32_t JoinWithTimeout( uint32_t timeoutMS, bool & outTimedOut ); // TODO:B Remove this unsafe API // Debugging static void SetThreadName( const char * name ); diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp index 84799dc3e..95bea6cf3 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp @@ -41,19 +41,14 @@ void WorkerThread::Init() PROFILE_FUNCTION; // Start thread - Thread::ThreadHandle h = Thread::CreateThread( ThreadWrapperFunc, - "WorkerThread", - MEGABYTE, - this ); - ASSERT( h != nullptr ); - Thread::DetachThread( h ); - Thread::CloseHandle( h ); // we don't want to keep this, so free it now + m_Thread.Start( ThreadWrapperFunc, "WorkerThread", this, MEGABYTE ); } //------------------------------------------------------------------------------ WorkerThread::~WorkerThread() { ASSERT( m_Exited.Load() ); + m_Thread.Join(); } // InitTmpDir diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h index da05980a5..61ef5b3ad 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h @@ -8,6 +8,7 @@ #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" #include "Core/Process/Semaphore.h" +#include "Core/Process/Thread.h" #include "Core/Strings/AStackString.h" #include "Core/Strings/AString.h" @@ -49,6 +50,7 @@ class WorkerThread virtual void Main(); // signal to exit thread + Thread m_Thread; Atomic m_ShouldExit; Atomic m_Exited; uint16_t m_ThreadIndex; From 22b92f651ac98a287d995213f9b00c206ffd2bd0 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 19 Mar 2023 11:48:06 +1030 Subject: [PATCH 010/129] BFFFuzzer: Fix linking with xxHash3 --- Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff b/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff index 32609aaf9..b44292908 100644 --- a/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff +++ b/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff @@ -52,6 +52,7 @@ 'FBuildCore-Lib-$Platform$-$BuildConfigName$', 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/bfffuzzer$ExeExtension$' .LinkerOptions + ' -pthread -ldl -lrt' From 24742946ba083d3f239b71458984b5d9077a4238 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 19 Mar 2023 11:50:10 +1030 Subject: [PATCH 011/129] Various small performance improvements through use of xxHash3 [Improvement] Cache performance improved slightly for check, store and retrieve operations [Improvement] Toolchain management performance improved slightly when toolchain changes [Improvement] Performance of various build steps improved slightly [Improvement] Database validation improved slightly, reducing startup/shutdown time slightly - switch 64bit hashes to xxHash3 which is ~2x-3x faster than xxHash [Note] DB version changed [Note] Cache version changed --- Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp | 4 ++-- Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp | 10 +++++----- Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/DirectoryListNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp | 6 +++--- Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp | 6 +++--- Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp | 4 ++-- Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp | 2 +- .../Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp | 6 +++--- Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp | 2 +- Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp | 2 +- 15 files changed, 27 insertions(+), 27 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp index 21522c78b..2a0503c14 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp @@ -25,7 +25,7 @@ BFFFile::BFFFile( const char * fileName, const AString & fileContents ) , m_FileContents( fileContents ) , m_Once( false ) { - m_Hash = xxHash::Calc64( m_FileContents ); + m_Hash = xxHash3::Calc64( m_FileContents ); } // DESTRUCTOR @@ -68,7 +68,7 @@ bool BFFFile::Load( const AString & fileName, const BFFToken * token ) m_FileContents = Move( fileContents ); m_FileName = fileName; m_ModTime = FileIO::GetFileLastWriteTime( fileName ); - m_Hash = xxHash::Calc64( m_FileContents ); + m_Hash = xxHash3::Calc64( m_FileContents ); return true; } diff --git a/Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp b/Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp index 8828bcae3..9b8672149 100644 --- a/Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp +++ b/Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp @@ -16,7 +16,7 @@ AString & outCacheId ) { // cache version - bump if cache format is changed - const char cacheVersion( 'D' ); + const char cacheVersion( 'E' ); // format example: 2377DE32AB045A2D_FED872A1_AB62FEAA23498AAC-32A2B04375A2D7DE.7 outCacheId.Format( "%016" PRIX64 "_%08X_%016" PRIX64 "-%016" PRIX64 ".%c", diff --git a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp index 6bd8317b7..1c379ccfa 100644 --- a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp +++ b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp @@ -331,7 +331,7 @@ bool LightCache::Hash( ObjectNode * node, hashes.Append( file->m_ContentHash ); outIncludes.Append( file->m_FileName ); } - outSourceHash = xxHash::Calc64( hashes.Begin(), hashes.GetSize() * sizeof( uint64_t ) ); + outSourceHash = xxHash3::Calc64( hashes.Begin(), hashes.GetSize() * sizeof( uint64_t ) ); return true; } @@ -365,7 +365,7 @@ void LightCache::Parse( IncludedFile * file, FileStream & f ) // Store hash of file - file->m_ContentHash = xxHash::Calc64( fileContents ); + file->m_ContentHash = xxHash3::Calc64( fileContents ); const char * pos = fileContents.Get(); for (;;) @@ -503,7 +503,7 @@ bool LightCache::ParseDirective_Define( IncludedFile & file, const char * & pos if ( ParseIncludeString( pos, include, includeType ) == false ) { // Not an include. We only care that this exists (not what it resolves to) - file.m_NonIncludeDefines.Append( xxHash::Calc64( macroName ) ); + file.m_NonIncludeDefines.Append( xxHash3::Calc64( macroName ) ); return true; } @@ -649,7 +649,7 @@ void LightCache::ProcessInclude( const AString & include, IncludeType type ) // The macro was not defined as a valid include path // Is it defined at all? - const uint64_t hash = xxHash::Calc64( include ); + const uint64_t hash = xxHash3::Calc64( include ); bool foundNonIncludeDefine = false; for ( const IncludedFile * includedFile : m_AllIncludedFiles ) { @@ -858,7 +858,7 @@ const IncludedFile * LightCache::ProcessIncludeFromIncludePath( const AString & //------------------------------------------------------------------------------ const IncludedFile * LightCache::FileExists( const AString & fileName ) { - const uint64_t fileNameHash = xxHash::Calc64( fileName ); + const uint64_t fileNameHash = xxHash3::Calc64( fileName ); const uint64_t bucketIndex = LIGHTCACHE_HASH_TO_BUCKET( fileNameHash ); IncludedFileBucket & bucket = g_AllIncludedFiles[ bucketIndex ]; // Retrieve from shared cache diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp index 8e8b28d1b..ae63bf32b 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp @@ -190,7 +190,7 @@ CopyDirNode::~CopyDirNode() = default; ASSERT( cn->GetStamp() ); stamps.Append( cn->GetStamp() ); } - m_Stamp = xxHash::Calc64( &stamps[ 0 ], ( stamps.GetSize() * sizeof( uint64_t ) ) ); + m_Stamp = xxHash3::Calc64( &stamps[ 0 ], ( stamps.GetSize() * sizeof( uint64_t ) ) ); } return NODE_RESULT_OK; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp index f1534e216..783ac27b8 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp @@ -234,7 +234,7 @@ DirectoryListNode::~DirectoryListNode() = default; ms.Write( file.IsReadOnly() ); } } - m_Stamp = xxHash::Calc64( ms.GetData(), ms.GetSize() ); + m_Stamp = xxHash3::Calc64( ms.GetData(), ms.GetSize() ); } return NODE_RESULT_OK; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index 0f090e27a..fe46dcf36 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -306,7 +306,7 @@ NodeGraph::LoadResult NodeGraph::Load( ConstMemoryStream & stream, const char * return LoadResult::LOAD_ERROR; // error reading } - const uint64_t dataHash = xxHash::Calc64( mem.Get(), size ); + const uint64_t dataHash = xxHash3::Calc64( mem.Get(), size ); if ( dataHash == usedFiles[ i ].m_DataHash ) { // file didn't change, update stored timestamp to save time on the next run @@ -537,7 +537,7 @@ void NodeGraph::Save( MemoryStream & stream, const char* nodeGraphDBFile ) const char * data = static_cast( stream.GetDataMutable() ); const char * content = ( data + sizeof(NodeGraphHeader) ); const size_t remainingSize = ( stream.GetSize() - sizeof(NodeGraphHeader) ); - const uint64_t hash = xxHash::Calc64( content, remainingSize ); + const uint64_t hash = xxHash3::Calc64( content, remainingSize ); // Update hash in header NodeGraphHeader * headerToUpdate = reinterpret_cast( data ); @@ -1862,7 +1862,7 @@ bool NodeGraph::ReadHeaderAndUsedFiles( ConstMemoryStream & nodeGraphStream, con ASSERT( tell == sizeof( NodeGraphHeader ) ); // Stream should be after header const char* data = ( static_cast( nodeGraphStream.GetData() ) + tell ); const size_t remainingSize = ( nodeGraphStream.GetSize() - tell ); - const uint64_t hash = xxHash::Calc64( data, remainingSize ); + const uint64_t hash = xxHash3::Calc64( data, remainingSize ); if ( hash != ngh.GetContentHash() ) { return false; // DB is corrupt diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index f4a3c980a..2b5083001 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -63,7 +63,7 @@ class NodeGraphHeader } inline ~NodeGraphHeader() = default; - enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 168 }; + enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 169 }; bool IsValid() const; bool IsCompatibleVersion() const { return m_Version == NODE_GRAPH_CURRENT_VERSION; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index 1ae95d74a..a5b760176 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -574,7 +574,7 @@ ObjectListNode::~ObjectListNode() = default; ASSERT( on->GetStamp() ); stamps.Append( on->GetStamp() ); } - m_Stamp = xxHash::Calc64( &stamps[0], ( stamps.GetSize() * sizeof(uint64_t) ) ); + m_Stamp = xxHash3::Calc64( &stamps[0], ( stamps.GetSize() * sizeof(uint64_t) ) ); } return NODE_RESULT_OK; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp index cf32de103..0b139f1df 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp @@ -1334,7 +1334,7 @@ const AString & ObjectNode::GetCacheName( Job * job ) const // hash the pre-processed input data ASSERT( m_LightCacheKey || job->GetData() ); - const uint64_t preprocessedSourceKey = m_LightCacheKey ? m_LightCacheKey : xxHash::Calc64( job->GetData(), job->GetDataSize() ); + const uint64_t preprocessedSourceKey = m_LightCacheKey ? m_LightCacheKey : xxHash3::Calc64( job->GetData(), job->GetDataSize() ); ASSERT( preprocessedSourceKey ); // hash the build "environment" @@ -1408,7 +1408,7 @@ bool ObjectNode::RetrieveFromCache( Job * job ) uint64_t pchKey = 0; if ( IsCreatingPCH() && IsMSVC() ) { - pchKey = xxHash::Calc64( cacheData, cacheDataSize ); + pchKey = xxHash3::Calc64( cacheData, cacheDataSize ); } const uint32_t startDecompress = uint32_t( t.GetElapsedMS() ); @@ -1590,7 +1590,7 @@ void ObjectNode::WriteToCache_FromCompressedData( Job * job, // Dependent objects need to know the PCH key to be able to pull from the cache if ( IsCreatingPCH() && IsMSVC() ) { - m_PCHCacheKey = xxHash::Calc64( compressedData, compressedDataSize ); + m_PCHCacheKey = xxHash3::Calc64( compressedData, compressedDataSize ); } const uint32_t cachingTime = uint32_t( t.GetElapsedMS() ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp index 9755575a6..b6a85ba5d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp @@ -299,7 +299,7 @@ SLNNode::~SLNNode() = default; } // Record stamp representing the contents of the files - m_Stamp = xxHash::Calc64( sln ); + m_Stamp = xxHash3::Calc64( sln ); return NODE_RESULT_OK; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp index f9c929b34..fc52a1ec4 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp @@ -514,7 +514,7 @@ UnityNode::~UnityNode() m_UnityFileNames.Append( unityName ); } - stamps.Append( xxHash::Calc64( output.Get(), output.GetLength() ) ); + stamps.Append( xxHash3::Calc64( output.Get(), output.GetLength() ) ); // need to write the unity file? bool needToWrite = false; @@ -584,7 +584,7 @@ UnityNode::~UnityNode() // Calculate final hash to represent generation of Unity files ASSERT( stamps.GetSize() == m_NumUnityFilesToCreate ); - m_Stamp = xxHash::Calc64( &stamps[ 0 ], stamps.GetSize() * sizeof( uint64_t ) ); + m_Stamp = xxHash3::Calc64( &stamps[ 0 ], stamps.GetSize() * sizeof( uint64_t ) ); // Track "nounity" status in the lest significant bit if (noUnity) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp index b6bfef1b5..1c8afe2d4 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp @@ -311,7 +311,7 @@ VCXProjectNode::~VCXProjectNode() = default; } // Record stamp representing the contents of the files - m_Stamp = xxHash::Calc64( project ) + xxHash::Calc64( filters ); + m_Stamp = xxHash3::Calc64( project ) + xxHash3::Calc64( filters ); return NODE_RESULT_OK; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp index 970dc3222..4d9dbf973 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp @@ -251,7 +251,7 @@ XCodeProjectNode::~XCodeProjectNode() = default; } // Combine hash - stamp += xxHash::Calc64( output ); + stamp += xxHash3::Calc64( output ); } // Get folder containing project.pbxproj @@ -285,7 +285,7 @@ XCodeProjectNode::~XCodeProjectNode() = default; } // Combine hash - stamp += xxHash::Calc64( output ); + stamp += xxHash3::Calc64( output ); } // Generate .xcscheme file @@ -306,7 +306,7 @@ XCodeProjectNode::~XCodeProjectNode() = default; } // Combine hash - stamp += xxHash::Calc64( output ); + stamp += xxHash3::Calc64( output ); } // Record stamp representing the contents of the files diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp index b7340bfc7..f0ee99a2a 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp @@ -216,7 +216,7 @@ bool ToolManifest::DoBuild( const Dependencies & dependencies ) *pos = xxHash::Calc32( relativePath ); ++pos; } - m_ToolId = xxHash::Calc64( mem, memSize ); + m_ToolId = xxHash3::Calc64( mem, memSize ); FREE( mem ); // update time stamp (most recent file in manifest) diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp index 3b42387b0..8fba02ce2 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp @@ -859,7 +859,7 @@ void TestGraph::DBVersionChanged() const { // Generate a fake old version headers NodeGraphHeader header; - header.SetContentHash( xxHash::Calc64( "", 0 ) ); + header.SetContentHash( xxHash3::Calc64( "", 0 ) ); MemoryStream ms; ms.WriteBuffer( &header, sizeof( header ) ); From 7119e67d9759e9c4276850e7e419d98425531123 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 26 Mar 2023 13:22:21 +1030 Subject: [PATCH 012/129] BFFVariables: Variables track their origin to facilitate improved errors in the future - We keep a pointer to the BFFToken responsible for the creation of the variable --- .../Tools/FBuild/FBuildCore/BFF/BFFParser.cpp | 80 +++++++++---------- Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h | 6 +- .../FBuild/FBuildCore/BFF/BFFStackFrame.cpp | 57 ++++++++----- .../FBuild/FBuildCore/BFF/BFFStackFrame.h | 17 +++- .../FBuild/FBuildCore/BFF/BFFVariable.cpp | 51 ++++++++---- .../Tools/FBuild/FBuildCore/BFF/BFFVariable.h | 19 +++-- .../BFF/Functions/FunctionForEach.cpp | 6 +- .../BFF/Functions/FunctionUsing.cpp | 4 +- .../FBuildCore/BFF/Tokenizer/BFFToken.cpp | 9 +++ .../FBuildCore/BFF/Tokenizer/BFFToken.h | 7 ++ .../FBuildTest/Tests/TestNodeReflection.cpp | 60 +++++++------- .../FBuildTest/Tests/TestVariableStack.cpp | 13 +-- 12 files changed, 202 insertions(+), 127 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.cpp index 8f00853b2..b3dc50fb7 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.cpp @@ -389,7 +389,7 @@ bool BFFParser::ParseVariableDeclaration( BFFTokenRange & iter, const AString & { newVal = i; } - return StoreVariableInt( varName, newVal, frame ); + return StoreVariableInt( varName, opToken, newVal, frame ); } else if ( rhsToken->IsBooelan() ) { @@ -415,7 +415,7 @@ bool BFFParser::ParseVariableDeclaration( BFFTokenRange & iter, const AString & iter++; // Consume the rhs - return StoreVariableBool( varName, rhsToken->GetBoolean(), frame ); + return StoreVariableBool( varName, rhsToken, rhsToken->GetBoolean(), frame ); } else if ( rhsToken->IsVariable() ) { @@ -695,15 +695,15 @@ bool BFFParser::ParseUserFunctionCall( BFFTokenRange & iter, const BFFUserFuncti { return false; } - BFFStackFrame::SetVarString( argName, value, &frame ); + BFFStackFrame::SetVarString( argName, *arg, value, &frame ); } else if ( arg->IsBooelan() ) { - BFFStackFrame::SetVarBool( argName, arg->GetBoolean(), &frame ); + BFFStackFrame::SetVarBool( argName, *arg, arg->GetBoolean(), &frame ); } else if ( arg->IsNumber() ) { - BFFStackFrame::SetVarInt( argName, arg->GetValueInt(), &frame ); + BFFStackFrame::SetVarInt( argName, *arg, arg->GetValueInt(), &frame ); } else { @@ -734,7 +734,7 @@ bool BFFParser::ParseUserFunctionCall( BFFTokenRange & iter, const BFFUserFuncti } // Set in function frame with argument name - BFFStackFrame::SetVar( varSrc, argName, &frame ); + BFFStackFrame::SetVar( varSrc, *arg, argName, &frame ); } } @@ -871,7 +871,7 @@ bool BFFParser::StoreVariableString( const AString & name, finalValue.Replace( value.Get(), "" ); } - BFFStackFrame::SetVarString( name, finalValue, frame ); + BFFStackFrame::SetVarString( name, *opToken, finalValue, frame ); return true; } else if ( var->IsArrayOfStrings() || dstIsEmpty ) @@ -898,7 +898,7 @@ bool BFFParser::StoreVariableString( const AString & name, } } - BFFStackFrame::SetVarArrayOfStrings( name, finalValues, frame ); + BFFStackFrame::SetVarArrayOfStrings( name, *opToken, finalValues, frame ); return true; } else @@ -913,7 +913,7 @@ bool BFFParser::StoreVariableString( const AString & name, if ( ( var == nullptr ) || var->IsString() ) { // OK - asigning to a new variable or to a string - BFFStackFrame::SetVarString( name, value, frame ); + BFFStackFrame::SetVarString( name, *opToken, value, frame ); return true; } else if ( var->IsArrayOfStrings() || dstIsEmpty ) @@ -921,7 +921,7 @@ bool BFFParser::StoreVariableString( const AString & name, // OK - store new string as the single element of array StackArray values; values.Append( value ); - BFFStackFrame::SetVarArrayOfStrings( name, values, frame ); + BFFStackFrame::SetVarArrayOfStrings( name, *opToken, values, frame ); return true; } else @@ -1171,13 +1171,13 @@ bool BFFParser::StoreVariableArray( const AString & name, if ( varType == BFFVariable::VAR_ARRAY_OF_STRUCTS ) { // structs - BFFStackFrame::SetVarArrayOfStructs( name, structValues, frame ); + BFFStackFrame::SetVarArrayOfStructs( name, *opToken, structValues, frame ); } else { ASSERT( varType == BFFVariable::VAR_ARRAY_OF_STRINGS ); // strings - BFFStackFrame::SetVarArrayOfStrings( name, values, frame ); + BFFStackFrame::SetVarArrayOfStrings( name, *opToken, values, frame ); } return true; @@ -1227,26 +1227,26 @@ bool BFFParser::StoreVariableStruct( const AString & name, Array & structMembers = stackFrame.GetLocalVariables(); // Register this variable - BFFStackFrame::SetVarStruct( name, Move( structMembers ), frame ? frame : stackFrame.GetParent() ); + BFFStackFrame::SetVarStruct( name, *operatorToken, Move( structMembers ), frame ? frame : stackFrame.GetParent() ); return true; } // StoreVariableBool //------------------------------------------------------------------------------ -bool BFFParser::StoreVariableBool( const AString & name, bool value, BFFStackFrame * frame ) +bool BFFParser::StoreVariableBool( const AString & name, const BFFToken * token, bool value, BFFStackFrame * frame ) { // Register this variable - BFFStackFrame::SetVarBool( name, value, frame ); + BFFStackFrame::SetVarBool( name, *token, value, frame ); return true; } // StoreVariableInt //------------------------------------------------------------------------------ -bool BFFParser::StoreVariableInt( const AString & name, int value, BFFStackFrame * frame ) +bool BFFParser::StoreVariableInt( const AString & name, const BFFToken * token, int value, BFFStackFrame * frame ) { - BFFStackFrame::SetVarInt( name, value, frame ); + BFFStackFrame::SetVarInt( name, *token, value, frame ); return true; } @@ -1372,7 +1372,7 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken values.Append( varSrc->GetString() ); } - BFFStackFrame::SetVarArrayOfStrings( dstName, values, dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), values, dstFrame ); return true; } @@ -1390,7 +1390,7 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken } values.Append( varSrc ); - BFFStackFrame::SetVarArrayOfStructs( dstName, values, dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), values, dstFrame ); return true; } @@ -1403,12 +1403,12 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken const BFFVariable * var = ( dstFrame ? dstFrame : BFFStackFrame::GetCurrent() )->GetLocalVar( dstName ); if ( varDst != var ) { - BFFStackFrame::SetVarArrayOfStrings( dstName, varDst->GetArrayOfStrings(), dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), varDst->GetArrayOfStrings(), dstFrame ); } } else { - BFFStackFrame::SetVarArrayOfStrings( dstName, Array< AString >(), dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), Array< AString >(), dstFrame ); } return true; } @@ -1422,12 +1422,12 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken const BFFVariable * var = ( dstFrame ? dstFrame : BFFStackFrame::GetCurrent() )->GetLocalVar( dstName ); if ( varDst != var ) { - BFFStackFrame::SetVarArrayOfStructs( dstName, varDst->GetArrayOfStructs(), dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), varDst->GetArrayOfStructs(), dstFrame ); } } else { - BFFStackFrame::SetVarArrayOfStructs(dstName, Array< const BFFVariable * >(), dstFrame); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), Array< const BFFVariable * >(), dstFrame ); } return true; } @@ -1435,14 +1435,14 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken // ArrayOfStrings to empty array, assignment or concatenation if ( dstIsEmpty && srcType == BFFVariable::VAR_ARRAY_OF_STRINGS && !subtract ) { - BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetArrayOfStrings(), dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), varSrc->GetArrayOfStrings(), dstFrame ); return true; } // ArrayOfStructs to empty array, assignment or concatenation if ( dstIsEmpty && srcType == BFFVariable::VAR_ARRAY_OF_STRUCTS && !subtract ) { - BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetArrayOfStructs(), dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), varSrc->GetArrayOfStructs(), dstFrame ); return true; } } @@ -1456,17 +1456,17 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken { AStackString< 2048 > finalValue(varDst->GetString()); finalValue += varSrc->GetString(); - BFFStackFrame::SetVarString( dstName, finalValue, dstFrame ); + BFFStackFrame::SetVarString( dstName, varSrc->GetToken(), finalValue, dstFrame ); } else if ( subtract ) { AStackString< 2048 > finalValue(varDst->GetString()); finalValue.Replace( varSrc->GetString().Get(), "" ); - BFFStackFrame::SetVarString( dstName, finalValue, dstFrame ); + BFFStackFrame::SetVarString( dstName, varSrc->GetToken(), finalValue, dstFrame ); } else { - BFFStackFrame::SetVarString( dstName, varSrc->GetString(), dstFrame ); + BFFStackFrame::SetVarString( dstName, varSrc->GetToken(), varSrc->GetString(), dstFrame ); } return true; } @@ -1480,11 +1480,11 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken values.SetCapacity( num ); values.Append( varDst->GetArrayOfStrings() ); values.Append( varSrc->GetArrayOfStrings() ); - BFFStackFrame::SetVarArrayOfStrings( dstName, values, dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), values, dstFrame ); } else { - BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetArrayOfStrings(), dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), varSrc->GetArrayOfStrings(), dstFrame ); } return true; } @@ -1498,11 +1498,11 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken values.SetCapacity( num ); values.Append( varDst->GetArrayOfStructs() ); values.Append( varSrc->GetArrayOfStructs() ); - BFFStackFrame::SetVarArrayOfStructs( dstName, values, dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), values, dstFrame ); } else { - BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetArrayOfStructs(), dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), varSrc->GetArrayOfStructs(), dstFrame ); } return true; } @@ -1522,12 +1522,12 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken { newVal = varSrc->GetInt(); } - return StoreVariableInt( dstName, newVal, dstFrame ); + return StoreVariableInt( dstName, &varSrc->GetToken(), newVal, dstFrame ); } if ( ( srcType == BFFVariable::VAR_BOOL ) && !concat && !subtract ) { - return StoreVariableBool( dstName, varSrc->GetBool(), dstFrame ); + return StoreVariableBool( dstName, &varSrc->GetToken(), varSrc->GetBool(), dstFrame ); } if ( ( srcType == BFFVariable::VAR_STRUCT ) && !subtract ) @@ -1544,7 +1544,7 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken else { // Register this variable - BFFStackFrame::SetVarStruct( dstName, srcMembers, dstFrame ); + BFFStackFrame::SetVarStruct( dstName, varSrc->GetToken(), srcMembers, dstFrame ); } return true; } @@ -1665,7 +1665,7 @@ void BFFParser::CreateBuiltInVariables() { AStackString<> varName( "._WORKING_DIR_" ); ASSERT( BFFStackFrame::GetVarAny( AStackString<>( varName.Get() + 1 ) ) == nullptr ); - BFFStackFrame::SetVarString( varName, FBuild::Get().GetWorkingDir(), &m_BaseStackFrame ); + BFFStackFrame::SetVarString( varName, BFFToken::GetBuiltInToken(), FBuild::Get().GetWorkingDir(), &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } @@ -1673,7 +1673,7 @@ void BFFParser::CreateBuiltInVariables() { AStackString<> varName( "._FASTBUILD_VERSION_STRING_" ); ASSERT( BFFStackFrame::GetVarAny( AStackString<>( varName.Get() + 1 ) ) == nullptr ); - BFFStackFrame::SetVarString( varName, AStackString<>(FBUILD_VERSION_STRING), &m_BaseStackFrame ); + BFFStackFrame::SetVarString( varName, BFFToken::GetBuiltInToken(), AStackString<>(FBUILD_VERSION_STRING), &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } @@ -1681,7 +1681,7 @@ void BFFParser::CreateBuiltInVariables() { AStackString<> varName( "._FASTBUILD_VERSION_" ); ASSERT( BFFStackFrame::GetVarAny( AStackString<>( varName.Get() + 1 ) ) == nullptr ); - BFFStackFrame::SetVarInt( varName, (int32_t)FBUILD_VERSION, &m_BaseStackFrame ); + BFFStackFrame::SetVarInt( varName, BFFToken::GetBuiltInToken(), (int32_t)FBUILD_VERSION, &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } @@ -1691,7 +1691,7 @@ void BFFParser::CreateBuiltInVariables() ASSERT( BFFStackFrame::GetVarAny( AStackString<>( varName.Get() + 1 ) ) == nullptr ); AStackString<> exeName; Env::GetExePath( exeName ); - BFFStackFrame::SetVarString( varName, exeName, &m_BaseStackFrame ); + BFFStackFrame::SetVarString( varName, BFFToken::GetBuiltInToken(), exeName, &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } } @@ -1728,7 +1728,7 @@ void BFFParser::SetBuiltInVariable_CurrentBFFDir( const BFFFile & file ) // Set the variable - always in the base scope const AStackString<> varName( "._CURRENT_BFF_DIR_" ); - BFFStackFrame::SetVarString( varName, relativePath, &m_BaseStackFrame ); + BFFStackFrame::SetVarString( varName, BFFToken::GetBuiltInToken(), relativePath, &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h b/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h index 92bf577a4..70c5a73a1 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h @@ -6,7 +6,6 @@ //------------------------------------------------------------------------------ #include "Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h" #include "Tools/FBuild/FBuildCore/BFF/LinkerNodeFileExistsCache.h" -#include "Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h" #include "Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.h" #include "Core/Env/Assert.h" @@ -17,6 +16,7 @@ // Forward Declarations //------------------------------------------------------------------------------ class BFFFile; +class BFFToken; class BFFTokenRange; class BFFUserFunction; class FileStream; @@ -76,8 +76,8 @@ class BFFParser bool StoreVariableString( const AString & name, const BFFToken * rhsString, const BFFToken * operatorToken, BFFStackFrame * frame ); bool StoreVariableArray( const AString & name, const BFFTokenRange & tokenRange, const BFFToken * operatorIter, BFFStackFrame * frame ); bool StoreVariableStruct( const AString & name, const BFFTokenRange & tokenRange, const BFFToken * operatorToken, BFFStackFrame * frame ); - bool StoreVariableBool( const AString & name, bool value, BFFStackFrame * frame ); - bool StoreVariableInt( const AString & name, int value, BFFStackFrame * frame ); + bool StoreVariableBool( const AString & name, const BFFToken * token, bool value, BFFStackFrame * frame ); + bool StoreVariableInt( const AString & name, const BFFToken * token, int value, BFFStackFrame * frame ); bool StoreVariableToVariable( const AString & dstName, const BFFToken * rhsToken, const BFFToken * operatorToken, BFFStackFrame * dstFrame ); void CreateBuiltInVariables(); diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp index 7e1565a72..539eaf478 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp @@ -5,8 +5,13 @@ //------------------------------------------------------------------------------ #include "BFFStackFrame.h" #include "BFFVariable.h" + +#include "Tools/FBuild/FBuildCore/BFF/Functions/Function.h" +#include "Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h" + #include "Core/Mem/Mem.h" #include "Core/Strings/AStackString.h" +#include "Core/Tracing/Tracing.h" // /*static*/ BFFStackFrame * BFFStackFrame::s_StackHead = nullptr; @@ -62,6 +67,7 @@ void BFFStackFrame::DisconnectStackChain() // SetVarString //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarString( const AString & name, + const BFFToken & token, const AString & value, BFFStackFrame * frame ) { @@ -76,13 +82,14 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, value ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, value ) ); frame->m_Variables.Append( v ); } // SetVarArrayOfStrings //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarArrayOfStrings( const AString & name, + const BFFToken & token, const Array< AString > & values, BFFStackFrame * frame ) { @@ -97,13 +104,16 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, values ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, values ) ); frame->m_Variables.Append( v ); } // SetVarBool //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVarBool( const AString & name, bool value, BFFStackFrame * frame ) +/*static*/ void BFFStackFrame::SetVarBool( const AString & name, + const BFFToken & token, + bool value, + BFFStackFrame * frame ) { frame = frame ? frame : s_StackHead; ASSERT( frame ); @@ -116,13 +126,16 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, value ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, value ) ); frame->m_Variables.Append( v ); } // SetVarInt //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVarInt( const AString & name, int value, BFFStackFrame * frame ) +/*static*/ void BFFStackFrame::SetVarInt( const AString & name, + const BFFToken & token, + int value, + BFFStackFrame * frame ) { frame = frame ? frame : s_StackHead; ASSERT( frame ); @@ -135,13 +148,14 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, value ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, value ) ); frame->m_Variables.Append( v ); } // SetVarStruct //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarStruct( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & members, BFFStackFrame * frame ) { @@ -156,13 +170,14 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, members ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, members ) ); frame->m_Variables.Append( v ); } // SetVarStruct //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarStruct( const AString& name, + const BFFToken & token, Array && members, BFFStackFrame * frame ) { @@ -177,7 +192,7 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable* v = FNEW( BFFVariable( name, Move( members ) ) ); + BFFVariable* v = FNEW( BFFVariable( name, token, Move( members ) ) ); frame->m_Variables.Append( v ); } @@ -185,6 +200,7 @@ void BFFStackFrame::DisconnectStackChain() // SetVarArrayOfStructs //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarArrayOfStructs( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & structs, BFFStackFrame * frame ) { @@ -199,21 +215,26 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, structs, BFFVariable::VAR_ARRAY_OF_STRUCTS ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, structs, BFFVariable::VAR_ARRAY_OF_STRUCTS ) ); frame->m_Variables.Append( v ); } // SetVar //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVar( const BFFVariable * var, BFFStackFrame * frame ) +/*static*/ void BFFStackFrame::SetVar( const BFFVariable * var, + const BFFToken & token, + BFFStackFrame * frame ) { - return SetVar( var, var->GetName(), frame ); + SetVar( var, token, var->GetName(), frame ); } // SetVar //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVar( const BFFVariable * srcVar, const AString & dstName, BFFStackFrame * frame ) +/*static*/ void BFFStackFrame::SetVar( const BFFVariable * srcVar, + const BFFToken & token, + const AString & dstName, + BFFStackFrame * frame ) { frame = frame ? frame : s_StackHead; ASSERT( frame ); @@ -223,12 +244,12 @@ void BFFStackFrame::DisconnectStackChain() switch ( srcVar->GetType() ) { case BFFVariable::VAR_ANY: ASSERT( false ); break; - case BFFVariable::VAR_STRING: SetVarString( dstName, srcVar->GetString(), frame ); break; - case BFFVariable::VAR_BOOL: SetVarBool( dstName, srcVar->GetBool(), frame ); break; - case BFFVariable::VAR_ARRAY_OF_STRINGS: SetVarArrayOfStrings( dstName, srcVar->GetArrayOfStrings(), frame ); break; - case BFFVariable::VAR_INT: SetVarInt( dstName, srcVar->GetInt(), frame ); break; - case BFFVariable::VAR_STRUCT: SetVarStruct( dstName, srcVar->GetStructMembers(), frame ); break; - case BFFVariable::VAR_ARRAY_OF_STRUCTS: SetVarArrayOfStructs( dstName, srcVar->GetArrayOfStructs(), frame ); break; + case BFFVariable::VAR_STRING: SetVarString( dstName, token, srcVar->GetString(), frame ); break; + case BFFVariable::VAR_BOOL: SetVarBool( dstName, token, srcVar->GetBool(), frame ); break; + case BFFVariable::VAR_ARRAY_OF_STRINGS: SetVarArrayOfStrings( dstName, token, srcVar->GetArrayOfStrings(), frame ); break; + case BFFVariable::VAR_INT: SetVarInt( dstName, token, srcVar->GetInt(), frame ); break; + case BFFVariable::VAR_STRUCT: SetVarStruct( dstName, token, srcVar->GetStructMembers(), frame ); break; + case BFFVariable::VAR_ARRAY_OF_STRUCTS: SetVarArrayOfStructs( dstName, token, srcVar->GetArrayOfStructs(), frame ); break; case BFFVariable::MAX_VAR_TYPES: ASSERT( false ); break; } } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h index 5fd8aeafc..73518f98d 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h @@ -10,6 +10,7 @@ // Forward Declarations //------------------------------------------------------------------------------ class AString; +class Function; // BFFStackFrame //------------------------------------------------------------------------------ @@ -23,30 +24,42 @@ class BFFStackFrame // set the value of a variable static void SetVarString( const AString & name, + const BFFToken & token, const AString & value, BFFStackFrame * frame ); static void SetVarArrayOfStrings( const AString & name, + const BFFToken & token, const Array< AString > & values, BFFStackFrame * frame ); static void SetVarBool( const AString & name, + const BFFToken & token, bool value, BFFStackFrame * frame ); static void SetVarInt( const AString & name, + const BFFToken & token, int value, BFFStackFrame * frame ); static void SetVarStruct( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & members, BFFStackFrame * frame ); static void SetVarStruct( const AString & name, + const BFFToken & token, Array && members, BFFStackFrame * frame ); static void SetVarArrayOfStructs( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & structs, BFFStackFrame * frame ); // set from an existing variable - static void SetVar( const BFFVariable * var, BFFStackFrame * frame ); - static void SetVar( const BFFVariable * srcVar, const AString & dstName, BFFStackFrame * frame ); + static void SetVar( const BFFVariable * var, + const BFFToken & token, + BFFStackFrame * frame ); + static void SetVar( const BFFVariable * srcVar, + const BFFToken & token, + const AString & dstName, + BFFStackFrame * frame ); // set from two existing variable static BFFVariable * ConcatVars( const AString & name, diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp index 1f412dda1..b4090e2fb 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp @@ -24,9 +24,10 @@ // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, VarType type ) +BFFVariable::BFFVariable( const AString & name, const BFFToken & token, VarType type ) : m_Name( name ) , m_Type( type ) + , m_Token( token ) { } @@ -35,6 +36,7 @@ BFFVariable::BFFVariable( const AString & name, VarType type ) BFFVariable::BFFVariable( const BFFVariable & other ) : m_Name( other.m_Name ) , m_Type( other.m_Type ) + , m_Token( other.m_Token ) { switch( m_Type ) { @@ -51,46 +53,61 @@ BFFVariable::BFFVariable( const BFFVariable & other ) // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, const AString & value ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + const AString & value ) : m_Name( name ) , m_Type( VAR_STRING ) , m_StringValue( value ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, bool value ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + bool value ) : m_Name( name ) , m_Type( VAR_BOOL ) , m_BoolValue( value ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, const Array< AString > & values ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + const Array< AString > & values ) : m_Name( name ) , m_Type( VAR_ARRAY_OF_STRINGS ) , m_ArrayValues( values ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, int32_t i ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + int32_t i ) : m_Name( name ) , m_Type( VAR_INT ) , m_IntValue( i ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, const Array< const BFFVariable * > & values ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + const Array< const BFFVariable * > & values ) : m_Name( name ) , m_Type( VAR_STRUCT ) , m_SubVariables( values.GetSize(), true ) + , m_Token( token ) { SetValueStruct( values ); } @@ -98,21 +115,25 @@ BFFVariable::BFFVariable( const AString & name, const Array< const BFFVariable * // CONSTRUCTOR (&&) //------------------------------------------------------------------------------ BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, Array && values ) : m_Name( name ) , m_Type( VAR_STRUCT ) , m_SubVariables( Move( values ) ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & structs, VarType type ) // type for disambiguation : m_Name( name ) , m_Type( VAR_ARRAY_OF_STRUCTS ) , m_SubVariables( structs.GetSize(), true ) + , m_Token( token ) { // type for disambiguation only - sanity check it's the right type ASSERT( type == VAR_ARRAY_OF_STRUCTS ); (void)type; @@ -288,7 +309,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF values.Append( varDst->GetArrayOfStrings() ); } values.Append( varSrc->GetString() ); - BFFVariable *result = FNEW(BFFVariable(dstName, values)); + BFFVariable * result = FNEW( BFFVariable( dstName, m_Token, values ) ); return result; } // Struct to ArrayOfStructs or empty array concatenation @@ -304,7 +325,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF } values.Append( varSrc ); - BFFVariable *result = FNEW( BFFVariable( dstName, values, VAR_ARRAY_OF_STRUCTS ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, m_Token, values, VAR_ARRAY_OF_STRUCTS ) ); return result; } @@ -313,7 +334,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF ( ( ( srcType == BFFVariable::VAR_ARRAY_OF_STRUCTS ) || ( srcType == BFFVariable::VAR_ARRAY_OF_STRINGS ) ) && dstIsEmpty ) ) { const BFFVariable * src = srcIsEmpty ? varDst : varSrc; - BFFVariable * result = FNEW( BFFVariable( dstName, src->m_Type ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, m_Token, src->m_Type ) ); if ( src->m_Type == BFFVariable::VAR_ARRAY_OF_STRINGS ) { result->SetValueArrayOfStrings( src->GetArrayOfStrings() ); @@ -342,7 +363,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF finalValue = varDst->GetString(); finalValue += varSrc->GetString(); - BFFVariable *result = FNEW( BFFVariable( dstName, finalValue ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, varSrc->m_Token, finalValue ) ); return result; } @@ -354,7 +375,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF values.Append( varDst->GetArrayOfStrings() ); values.Append( varSrc->GetArrayOfStrings() ); - BFFVariable *result = FNEW( BFFVariable( dstName, values ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, varSrc->m_Token, values ) ); return result; } @@ -366,7 +387,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF values.Append( varDst->GetArrayOfStructs() ); values.Append( varSrc->GetArrayOfStructs() ); - BFFVariable * result = FNEW(BFFVariable( dstName, values, VAR_ARRAY_OF_STRUCTS ) ); + BFFVariable * result = FNEW(BFFVariable( dstName, varSrc->m_Token, values, VAR_ARRAY_OF_STRUCTS ) ); return result; } @@ -375,7 +396,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF int newVal( varSrc->GetInt() ); newVal += varDst->GetInt(); - BFFVariable * result = FNEW( BFFVariable( dstName, newVal ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, varSrc->m_Token, newVal ) ); return result; } @@ -385,7 +406,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF bool newVal( varSrc->GetBool() ); newVal |= varDst->GetBool(); - BFFVariable * result = FNEW( BFFVariable( dstName, newVal ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, varSrc->m_Token, newVal ) ); return result; } @@ -394,7 +415,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF const Array< const BFFVariable * > & srcMembers = varSrc->GetStructMembers(); const Array< const BFFVariable * > & dstMembers = varDst->GetStructMembers(); - BFFVariable * const result = FNEW( BFFVariable( dstName, BFFVariable::VAR_STRUCT ) ); + BFFVariable * const result = FNEW( BFFVariable( dstName, varSrc->m_Token, BFFVariable::VAR_STRUCT ) ); result->m_SubVariables.SetCapacity( srcMembers.GetSize() + dstMembers.GetSize() ); Array< BFFVariable * > & allMembers = result->m_SubVariables; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.h b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.h index 2848117b8..a5ca1d832 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.h @@ -68,19 +68,21 @@ class BFFVariable static const BFFVariable ** GetMemberByName( const AString & name, const Array< const BFFVariable * > & members ); + const BFFToken & GetToken() const { return m_Token; } + private: friend class BFFStackFrame; explicit BFFVariable( const BFFVariable & other ); - explicit BFFVariable( const AString & name, VarType type ); - explicit BFFVariable( const AString & name, const AString & value ); - explicit BFFVariable( const AString & name, bool value ); - explicit BFFVariable( const AString & name, const Array< AString > & values ); - explicit BFFVariable( const AString & name, int32_t i ); - explicit BFFVariable( const AString & name, const Array< const BFFVariable * > & values ); - explicit BFFVariable( const AString & name, Array && values ); - explicit BFFVariable( const AString & name, const Array< const BFFVariable * > & structs, VarType type ); // type for disambiguation + explicit BFFVariable( const AString & name, const BFFToken & token, VarType type ); + explicit BFFVariable( const AString & name, const BFFToken & token, const AString & value ); + explicit BFFVariable( const AString & name, const BFFToken & token, bool value ); + explicit BFFVariable( const AString & name, const BFFToken & token, const Array< AString > & values ); + explicit BFFVariable( const AString & name, const BFFToken & token, int32_t i ); + explicit BFFVariable( const AString & name, const BFFToken & token, const Array< const BFFVariable * > & values ); + explicit BFFVariable( const AString & name, const BFFToken & token, Array && values ); + explicit BFFVariable( const AString & name, const BFFToken & token, const Array< const BFFVariable * > & structs, VarType type ); // type for disambiguation ~BFFVariable(); BFFVariable & operator =( const BFFVariable & other ) = delete; @@ -104,6 +106,7 @@ class BFFVariable AString m_StringValue; Array< AString > m_ArrayValues; Array< BFFVariable * > m_SubVariables; // Used for struct members of arrays of structs + const BFFToken & m_Token; static const char * s_TypeNames[ MAX_VAR_TYPES ]; }; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionForEach.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionForEach.cpp index f41832146..92d1d88cf 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionForEach.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionForEach.cpp @@ -38,7 +38,7 @@ FunctionForEach::FunctionForEach() //------------------------------------------------------------------------------ /*virtual*/ bool FunctionForEach::ParseFunction( NodeGraph & /*nodeGraph*/, BFFParser & parser, - const BFFToken * /*functionNameStart*/, + const BFFToken * functionNameStart, const BFFTokenRange & headerRange, const BFFTokenRange & bodyRange ) const { @@ -189,11 +189,11 @@ FunctionForEach::FunctionForEach() { if ( arrayVars[ j ]->GetType() == BFFVariable::VAR_ARRAY_OF_STRINGS ) { - BFFStackFrame::SetVarString( localNames[ j ], arrayVars[ j ]->GetArrayOfStrings()[ i ], &loopStackFrame ); + BFFStackFrame::SetVarString( localNames[ j ], *functionNameStart, arrayVars[ j ]->GetArrayOfStrings()[ i ], &loopStackFrame ); } else if ( arrayVars[ j ]->GetType() == BFFVariable::VAR_ARRAY_OF_STRUCTS ) { - BFFStackFrame::SetVarStruct( localNames[ j ], arrayVars[ j ]->GetArrayOfStructs()[ i ]->GetStructMembers(), &loopStackFrame ); + BFFStackFrame::SetVarStruct( localNames[ j ], *functionNameStart, arrayVars[ j ]->GetArrayOfStructs()[ i ]->GetStructMembers(), &loopStackFrame ); } else { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp index f2ec822b8..e703eb173 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp @@ -108,12 +108,12 @@ FunctionUsing::FunctionUsing() sameNameMember = member; continue; } - BFFStackFrame::SetVar( member, frame ); + BFFStackFrame::SetVar( member, *varToken, frame ); } if ( sameNameMember != nullptr ) { - BFFStackFrame::SetVar( sameNameMember, frame ); + BFFStackFrame::SetVar( sameNameMember, *varToken, frame ); } if ( headerIter.IsAtEnd() == false ) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.cpp index 3ac5ab352..8a146ba0b 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.cpp @@ -7,6 +7,15 @@ #include +// Static Data +//------------------------------------------------------------------------------ +/*static*/ const BFFFile BFFToken::s_BuiltInFile( "", AString::GetEmpty() ); +/*static*/ const BFFToken BFFToken::s_BuiltInToken( s_BuiltInFile, + s_BuiltInFile.GetSourceFileContents().Get(), + BFFTokenType::Invalid, + s_BuiltInFile.GetSourceFileContents().Get(), + s_BuiltInFile.GetSourceFileContents().Get() ); + // CONSTRUCTOR //------------------------------------------------------------------------------ BFFToken::BFFToken( const BFFFile & file, diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h index 884c7b9df..19b67b11f 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h @@ -81,6 +81,9 @@ class BFFToken void GetPosInfo( uint32_t & outLine, uint32_t & outColumn, const char * & outLineStart ) const; + // Some variables come from built-in declarations so we need a proxy BFFToken for those + static const BFFToken & GetBuiltInToken() { return s_BuiltInToken; } + private: BFFTokenType m_Type; bool m_Boolean = false; @@ -88,6 +91,10 @@ class BFFToken AString m_String; const BFFFile & m_BFFFile; const char * m_SourcePos = nullptr; + + // Static Data + static const BFFFile s_BuiltInFile; + static const BFFToken s_BuiltInToken; }; //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp index 5f1e4a348..ce7c0277e 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp @@ -285,7 +285,7 @@ void TestNodeReflection::String_Optional_Set() const TestHelper helper( new Node_String_Optional ); // Set string - helper.m_Frame.SetVarString( AStackString<>( ".String" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".String" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check string was set TEST_ASSERT( helper.Populate() ); @@ -299,7 +299,7 @@ void TestNodeReflection::String_Optional_Empty() const TestHelper helper( new Node_String_Optional ); // Set string to empty - helper.m_Frame.SetVarString( AStackString<>( ".String" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".String" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Check ok (setting empty on optional property is ok) TEST_ASSERT( helper.Populate() ); @@ -333,7 +333,7 @@ void TestNodeReflection::String_Required_Set() const TestHelper helper( new Node_String_Required ); // Set string - helper.m_Frame.SetVarString( AStackString<>( ".String" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".String" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check string was set TEST_ASSERT( helper.Populate() ); @@ -347,7 +347,7 @@ void TestNodeReflection::String_Required_Empty() const TestHelper helper( new Node_String_Required ); // Set string to empty - helper.m_Frame.SetVarString( AStackString<>( ".String" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".String" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Check for failure (required strings can't be empty) TEST_ASSERT( helper.Populate() == false ); @@ -384,7 +384,7 @@ void TestNodeReflection::ArrayOfStrings_Optional_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check array was set TEST_ASSERT( helper.Populate() ); @@ -400,7 +400,7 @@ void TestNodeReflection::ArrayOfStrings_Optional_Empty() const // Set array to empty Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Check ok (setting empty on optional property is ok) TEST_ASSERT( helper.Populate() ); @@ -417,7 +417,7 @@ void TestNodeReflection::ArrayOfStrings_Optional_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -453,7 +453,7 @@ void TestNodeReflection::ArrayOfStrings_Required_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check array was set TEST_ASSERT( helper.Populate() ); @@ -469,7 +469,7 @@ void TestNodeReflection::ArrayOfStrings_Required_Empty() const // Set array to empty Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Check for failure (can't set empty array if property is required) TEST_ASSERT( helper.Populate() == false ); @@ -486,7 +486,7 @@ void TestNodeReflection::ArrayOfStrings_Required_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -519,7 +519,7 @@ void TestNodeReflection::MetaFile_String_Optional_Set() const TestHelper helper( new Node_MetaFile_String_Optional ); // Push a string - helper.m_Frame.SetVarString( AStackString<>( ".File" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".File" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -534,7 +534,7 @@ void TestNodeReflection::MetaFile_String_Optional_Empty() const TestHelper helper( new Node_MetaFile_String_Optional ); // Push an empty string - helper.m_Frame.SetVarString( AStackString<>( ".File" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".File" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Ok for property to be empty because it is optional TEST_ASSERT( helper.Populate() == true ); @@ -567,7 +567,7 @@ void TestNodeReflection::MetaFile_String_Required_Set() const TestHelper helper( new Node_MetaFile_String_Required ); // Push a string - helper.m_Frame.SetVarString( AStackString<>( ".File" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".File" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -582,7 +582,7 @@ void TestNodeReflection::MetaFile_String_Required_Empty() const TestHelper helper( new Node_MetaFile_String_Required ); // Push an empty string - helper.m_Frame.SetVarString( AStackString<>( ".File" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".File" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Check that populating properties fails and that appropriate error is reported TEST_ASSERT( helper.Populate() == false ); @@ -617,7 +617,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Optional_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check the property was set and converted to a full paths TEST_ASSERT( helper.Populate() == true ); @@ -634,7 +634,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Optional_Empty() const // Set string array with an empty element in in Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Ok for property to be empty because it is optional TEST_ASSERT( helper.Populate() == true ); @@ -650,7 +650,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Optional_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -686,7 +686,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Required_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -703,7 +703,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Required_Empty() const // Set string array with an empty element in in Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Check that populating properties fails and that appropriate error is reported TEST_ASSERT( helper.Populate() == false ); @@ -720,7 +720,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Required_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -753,7 +753,7 @@ void TestNodeReflection::MetaPath_String_Optional_Set() const TestHelper helper( new Node_MetaPath_String_Optional ); // Push a string - helper.m_Frame.SetVarString( AStackString<>( ".Path" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".Path" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -768,7 +768,7 @@ void TestNodeReflection::MetaPath_String_Optional_Empty() const TestHelper helper( new Node_MetaPath_String_Optional ); // Push an empty string - helper.m_Frame.SetVarString( AStackString<>( ".Path" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".Path" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Ok for property to be empty because it is optional TEST_ASSERT( helper.Populate() == true ); @@ -801,7 +801,7 @@ void TestNodeReflection::MetaPath_String_Required_Set() const TestHelper helper( new Node_MetaPath_String_Required ); // Push a string - helper.m_Frame.SetVarString( AStackString<>( ".Path" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".Path" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -816,7 +816,7 @@ void TestNodeReflection::MetaPath_String_Required_Empty() const TestHelper helper( new Node_MetaPath_String_Required ); // Push an empty string - helper.m_Frame.SetVarString( AStackString<>( ".Path" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".Path" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Check that populating properties fails and that appropriate error is reported TEST_ASSERT( helper.Populate() == false ); @@ -851,7 +851,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Optional_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check the property was set and converted to a full paths TEST_ASSERT( helper.Populate() == true ); @@ -868,7 +868,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Optional_Empty() const // Set string array with an empty element in in Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Ok for property to be empty because it is optional TEST_ASSERT( helper.Populate() == true ); @@ -884,7 +884,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Optional_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -920,7 +920,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Required_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -937,7 +937,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Required_Empty() const // Set string array with an empty element in in Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Check that populating properties fails and that appropriate error is reported TEST_ASSERT( helper.Populate() == false ); @@ -954,7 +954,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Required_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestVariableStack.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestVariableStack.cpp index 31825ec5d..d87ea18e7 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestVariableStack.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestVariableStack.cpp @@ -7,6 +7,7 @@ #include "Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h" #include "Tools/FBuild/FBuildCore/BFF/BFFVariable.h" +#include "Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h" #include "Core/Strings/AStackString.h" @@ -46,7 +47,7 @@ void TestVariableStack::TestStackFramesAdditional() const { // a stack frame with a variable BFFStackFrame sf1; - BFFStackFrame::SetVarString( AStackString<>( "myVarA" ), AStackString<>( "valueA" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVarA" ), BFFToken::GetBuiltInToken(), AStackString<>( "valueA" ), nullptr ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarA" ) ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarA" )->GetString() == "valueA" ); @@ -54,7 +55,7 @@ void TestVariableStack::TestStackFramesAdditional() const // another stack frame { BFFStackFrame sf2; - BFFStackFrame::SetVarString( AStackString<>( "myVarB" ), AStackString<>( "valueB" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVarB" ), BFFToken::GetBuiltInToken(), AStackString<>( "valueB" ), nullptr ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarA" ) ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarA" )->GetString() == "valueA" ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarB" ) ); @@ -73,7 +74,7 @@ void TestVariableStack::TestStackFramesOverride() const { // a stack frame with a variable BFFStackFrame sf1; - BFFStackFrame::SetVarString( AStackString<>( "myVar" ), AStackString<>( "originalValue" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVar" ), BFFToken::GetBuiltInToken(), AStackString<>( "originalValue" ), nullptr ); TEST_ASSERT( BFFStackFrame::GetVar( "myVar" ) ); TEST_ASSERT( BFFStackFrame::GetVar( "myVar" )->GetString() == "originalValue" ); @@ -82,7 +83,7 @@ void TestVariableStack::TestStackFramesOverride() const { // which replaces the same variable BFFStackFrame sf2; - BFFStackFrame::SetVarString( AStackString<>( "myVar" ), AStackString<>( "replacedValue" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVar" ), BFFToken::GetBuiltInToken(), AStackString<>( "replacedValue" ), nullptr ); // we should get the replaced value TEST_ASSERT( BFFStackFrame::GetVar( "myVar" ) ); @@ -102,7 +103,7 @@ void TestVariableStack::TestStackFramesParent() const // a stack frame with a variable BFFStackFrame sf1; - BFFStackFrame::SetVarString( AStackString<>( "myVar" ), AStackString<>( "originalValue" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVar" ), BFFToken::GetBuiltInToken(), AStackString<>( "originalValue" ), nullptr ); TEST_ASSERT( BFFStackFrame::GetVar( "myVar", &sf1 ) ); TEST_ASSERT( BFFStackFrame::GetVar( "myVar", &sf1 )->GetString() == "originalValue" ); @@ -118,7 +119,7 @@ void TestVariableStack::TestStackFramesParent() const TEST_ASSERT( BFFStackFrame::GetVar( "myVar", &sf2 ) == nullptr ); // which replaces the same variable - BFFStackFrame::SetVarString( AStackString<>( "myVar" ), AStackString<>( "replacedValue" ), &sf2 ); + BFFStackFrame::SetVarString( AStackString<>( "myVar" ), BFFToken::GetBuiltInToken(), AStackString<>( "replacedValue" ), &sf2 ); // we should bet the original value TEST_ASSERT( BFFStackFrame::GetVar( "myVar", &sf1 ) ); From a033b1a757068174afb5116acff58e2a38da6bfa Mon Sep 17 00:00:00 2001 From: Pete Lewis Date: Sat, 8 Apr 2023 18:35:42 -0700 Subject: [PATCH 013/129] Fix Linux and MacOS file permissions: (#956) * SetExecutable: preserve existing RW permissions; just add X * SetReadOnly: when removing the flag, remove W entirely; when adding the flag, only set for USR * mkdir: Create directories with 0755 permissions --- Code/Core/FileIO/FileIO.cpp | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Code/Core/FileIO/FileIO.cpp b/Code/Core/FileIO/FileIO.cpp index 5addcfdb6..8689368c0 100644 --- a/Code/Core/FileIO/FileIO.cpp +++ b/Code/Core/FileIO/FileIO.cpp @@ -500,7 +500,7 @@ } #elif defined( __LINUX__ ) || defined( __APPLE__ ) umask( 0 ); // disable default creation mask // TODO:LINUX TODO:MAC Changes global program state; needs investigation - mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO; // TODO:LINUX TODO:MAC Check these permissions + mode_t mode = (S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); if ( mkdir( path.Get(), mode ) == 0 ) { return true; // created ok @@ -888,7 +888,8 @@ { return true; // can't even get the attributes, treat as not read only } - const bool currentlyReadOnly = !( ( s.st_mode & S_IWUSR ) == S_IWUSR ); // TODO:LINUX Is this the correct behaviour? + const unsigned int anyWriteBits = S_IWUSR | S_IWGRP | S_IWOTH; + const bool currentlyReadOnly = ( ( s.st_mode & anyWriteBits ) == 0 ); if ( readOnly == currentlyReadOnly ) { return true; // already in desired state @@ -897,13 +898,13 @@ // update writable flag if ( readOnly ) { - // remove writable flag - s.st_mode &= ( ~S_IWUSR ); // TODO:LINUX Is this the correct behaviour? + // remove writable flag for everyone + s.st_mode &= ~( S_IWUSR | S_IWGRP | S_IWOTH ); } else { - // add writable flag - s.st_mode |= S_IWUSR; // TODO:LINUX Is this the correct behaviour? + // add writable flag for just the user + s.st_mode |= S_IWUSR; } if ( chmod( fileName, s.st_mode ) == 0 ) @@ -948,11 +949,14 @@ #if defined( __LINUX__ ) || defined( __APPLE__ ) /*static*/ bool FileIO::SetExecutable( const char * fileName ) { - // rwxr-x--x (751) TODO:LINUX TODO:MAC Is this correct? - mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR | - S_IRGRP | S_IXGRP | - S_IXOTH; - if ( chmod( fileName, mode ) == 0 ) + struct stat s; + if ( lstat( fileName, &s ) != 0 ) + { + return false; // failed to stat + } + + s.st_mode |= S_IXUSR | S_IXGRP | S_IXOTH; + if ( chmod( fileName, s.st_mode ) == 0 ) { return true; } From 077bf93a163663da89ecc8475b57b30707cf5539 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 9 Apr 2023 11:24:53 +0930 Subject: [PATCH 014/129] Post-integration cleanup for Linux/OSX FileIO permissions fixes - minor style fixes --- Code/Core/FileIO/FileIO.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/Core/FileIO/FileIO.cpp b/Code/Core/FileIO/FileIO.cpp index 8689368c0..8cb5615d7 100644 --- a/Code/Core/FileIO/FileIO.cpp +++ b/Code/Core/FileIO/FileIO.cpp @@ -500,7 +500,7 @@ } #elif defined( __LINUX__ ) || defined( __APPLE__ ) umask( 0 ); // disable default creation mask // TODO:LINUX TODO:MAC Changes global program state; needs investigation - mode_t mode = (S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); + mode_t mode = ( S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH ); if ( mkdir( path.Get(), mode ) == 0 ) { return true; // created ok @@ -888,7 +888,7 @@ { return true; // can't even get the attributes, treat as not read only } - const unsigned int anyWriteBits = S_IWUSR | S_IWGRP | S_IWOTH; + const uint32_t anyWriteBits = S_IWUSR | S_IWGRP | S_IWOTH; const bool currentlyReadOnly = ( ( s.st_mode & anyWriteBits ) == 0 ); if ( readOnly == currentlyReadOnly ) { From 873b1ccf202259a407a0dbfbc604b329d86ef5d5 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 9 Apr 2023 12:07:46 +0930 Subject: [PATCH 015/129] Remove Linux GCC 7 from github actions CI - seems like either GCC7 or the version of ubuntu used causes the github action to be stuck in queue forever --- .github/workflows/Linux.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index c030e870b..88639f3c9 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -17,10 +17,6 @@ jobs: gcc: 10 clang: 12 os: ubuntu-20.04 - - cfg: Build - gcc: 7 - clang: 9 - os: ubuntu-18.04 - cfg: ASan gcc: 10 clang: 12 From 9e7808c7a598ff29f663768798d5865f1b6d4cea Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Apr 2023 14:38:40 +0930 Subject: [PATCH 016/129] Mutex: Small code cleanup (no functional changes) --- Code/Core/Process/Mutex.cpp | 20 ++++++-------------- Code/Core/Process/Mutex.h | 8 ++------ 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/Code/Core/Process/Mutex.cpp b/Code/Core/Process/Mutex.cpp index bc1047a6e..e662b42e5 100644 --- a/Code/Core/Process/Mutex.cpp +++ b/Code/Core/Process/Mutex.cpp @@ -10,8 +10,7 @@ #if defined( __WINDOWS__ ) #include "Core/Env/WindowsHeader.h" -#endif -#if defined( __LINUX__ ) || defined( __APPLE__ ) +#else #include #endif @@ -24,13 +23,11 @@ Mutex::Mutex() static_assert( __alignof( decltype( m_CriticalSection ) ) == __alignof( CRITICAL_SECTION ), "Unexpected __alignof(CRITICAL_SECTION)" ); VERIFY( InitializeCriticalSectionAndSpinCount( (CRITICAL_SECTION *)&m_CriticalSection, 100 ) ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) + #else pthread_mutexattr_t attributes; VERIFY( pthread_mutexattr_init( &attributes ) == 0 ); pthread_mutexattr_settype( &attributes, PTHREAD_MUTEX_RECURSIVE ); VERIFY( pthread_mutex_init( &m_Mutex, &attributes ) == 0 ); - #else - #error Unknown platform #endif } @@ -40,10 +37,8 @@ Mutex::~Mutex() { #if defined( __WINDOWS__ ) DeleteCriticalSection( (CRITICAL_SECTION *)&m_CriticalSection ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) - VERIFY( pthread_mutex_destroy( &m_Mutex ) == 0 ); #else - #error Unknown platform + VERIFY( pthread_mutex_destroy( &m_Mutex ) == 0 ); #endif } @@ -54,22 +49,19 @@ void Mutex::Lock() { #if defined( __WINDOWS__ ) EnterCriticalSection( (CRITICAL_SECTION *)&m_CriticalSection ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) - VERIFY( pthread_mutex_lock( &m_Mutex ) == 0 ); #else - #error Unknown platform + VERIFY( pthread_mutex_lock( &m_Mutex ) == 0 ); #endif } + // Unlock //------------------------------------------------------------------------------ void Mutex::Unlock() { #if defined( __WINDOWS__ ) LeaveCriticalSection( (CRITICAL_SECTION *)&m_CriticalSection ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) - VERIFY( pthread_mutex_unlock( &m_Mutex ) == 0 ); #else - #error Unknown platform + VERIFY( pthread_mutex_unlock( &m_Mutex ) == 0 ); #endif } PRAGMA_DISABLE_POP_MSVC diff --git a/Code/Core/Process/Mutex.h b/Code/Core/Process/Mutex.h index 4d26573c7..9ade1adda 100644 --- a/Code/Core/Process/Mutex.h +++ b/Code/Core/Process/Mutex.h @@ -23,13 +23,9 @@ class Mutex private: // do this to avoid including windows.h - #if defined ( WIN64 ) + #if defined( __WINDOWS__ ) uint64_t m_CriticalSection[ 5 ]; // CRITICAL_SECTION - #elif defined ( WIN32 ) - uint32_t m_CriticalSection[ 6 ]; // CRITICAL_SECTION - #endif - - #if defined( __LINUX__ ) || defined( __APPLE__ ) + #else pthread_mutex_t m_Mutex; #endif }; From c9794e17483ccf49e7307c3e0eb5551273bbbc8c Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Apr 2023 15:07:20 +0930 Subject: [PATCH 017/129] Mutex: Add TryLock functionality - speculative lock acquisition that returns immediately on failure --- Code/Core/CoreTest/Tests/TestMutex.cpp | 75 ++++++++++++++++++++++++++ Code/Core/Process/Mutex.cpp | 11 ++++ Code/Core/Process/Mutex.h | 5 +- 3 files changed, 89 insertions(+), 2 deletions(-) diff --git a/Code/Core/CoreTest/Tests/TestMutex.cpp b/Code/Core/CoreTest/Tests/TestMutex.cpp index f8d054374..1d2bea5af 100644 --- a/Code/Core/CoreTest/Tests/TestMutex.cpp +++ b/Code/Core/CoreTest/Tests/TestMutex.cpp @@ -8,6 +8,7 @@ #include "Core/Mem/Mem.h" #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" +#include "Core/Process/Semaphore.h" #include "Core/Process/Thread.h" // TestMutex @@ -20,6 +21,11 @@ class TestMutex : public TestGroup void TestConstruct() const; void TestLockUnlock() const; void TestMultiLockUnlock() const; + void TryLock() const; + void TryLockMultiple() const; + void TryLockMixed() const; + void TryLockFail() const; + static uint32_t TestLockFailThreadEntryFunction( void * data ); void TestExclusivity() const; struct TestExclusivityUserData @@ -47,6 +53,10 @@ REGISTER_TESTS_BEGIN( TestMutex ) REGISTER_TEST( TestConstruct ) REGISTER_TEST( TestLockUnlock ) REGISTER_TEST( TestMultiLockUnlock ) + REGISTER_TEST( TryLock ) + REGISTER_TEST( TryLockMultiple ) + REGISTER_TEST( TryLockMixed ) + REGISTER_TEST( TryLockFail ) REGISTER_TEST( TestExclusivity ) #if defined( __WINDOWS__ ) REGISTER_TEST( TestAlignment ) @@ -80,6 +90,71 @@ void TestMutex::TestMultiLockUnlock() const m.Unlock(); } +// TryLock +//------------------------------------------------------------------------------ +void TestMutex::TryLock() const +{ + // Ensure lock can be acquired multiple times + Mutex m; + TEST_ASSERT( m.TryLock() ); + m.Unlock(); +} + +// TryLockMultiple +//------------------------------------------------------------------------------ +void TestMutex::TryLockMultiple() const +{ + // Ensure lock can be acquired multiple times + Mutex m; + TEST_ASSERT( m.TryLock() ); + TEST_ASSERT( m.TryLock() ); + m.Unlock(); + m.Unlock(); +} + +// TryLockMixed +//------------------------------------------------------------------------------ +void TestMutex::TryLockMixed() const +{ + Mutex m; + // Lock then TryLock + { + m.Lock(); + TEST_ASSERT( m.TryLock() ); + m.Unlock(); + m.Unlock(); + } + // TryLock then Lock + { + TEST_ASSERT( m.TryLock() ); + m.Lock(); + m.Unlock(); + m.Unlock(); + } +} + +// TryLockFail +//------------------------------------------------------------------------------ +void TestMutex::TryLockFail() const +{ + Mutex m; + MutexHolder mh( m ); // Hold lock so thread cannot acquire it + + Thread t; + t.Start( TestLockFailThreadEntryFunction, "TryLockFail", &m ); + t.Join(); +} + +// TestLockFailThreadEntryFunction +//------------------------------------------------------------------------------ +/*static*/ uint32_t TestMutex::TestLockFailThreadEntryFunction( void * data ) +{ + // main thread should hold lock + Mutex * m = static_cast( data ); + TEST_ASSERT( m->TryLock() == false ); + return 0; +} + // TestExclusivity //------------------------------------------------------------------------------ void TestMutex::TestExclusivity() const diff --git a/Code/Core/Process/Mutex.cpp b/Code/Core/Process/Mutex.cpp index e662b42e5..fbf853028 100644 --- a/Code/Core/Process/Mutex.cpp +++ b/Code/Core/Process/Mutex.cpp @@ -54,6 +54,17 @@ void Mutex::Lock() #endif } +// TryLock +//------------------------------------------------------------------------------ +bool Mutex::TryLock() +{ + #if defined( __WINDOWS__ ) + return ( TryEnterCriticalSection( (CRITICAL_SECTION *)&m_CriticalSection ) != FALSE ); + #else + return ( pthread_mutex_trylock( &m_Mutex ) == 0 ); + #endif +} + // Unlock //------------------------------------------------------------------------------ void Mutex::Unlock() diff --git a/Code/Core/Process/Mutex.h b/Code/Core/Process/Mutex.h index 9ade1adda..8a4597832 100644 --- a/Code/Core/Process/Mutex.h +++ b/Code/Core/Process/Mutex.h @@ -18,8 +18,9 @@ class Mutex Mutex(); ~Mutex(); - void Lock(); - void Unlock(); + void Lock(); + [[nodiscard]] bool TryLock(); + void Unlock(); private: // do this to avoid including windows.h From 755d5be656ffe36cec30b4ed99b933eff7c4f858 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Apr 2023 16:42:09 +0930 Subject: [PATCH 018/129] [Fix] Prevent Worker over-requesting jobs when there are multiple Clients --- Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index 23c93695f..6c3552ac4 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -666,6 +666,12 @@ void Server::FindNeedyClients() cs->m_NumJobsRequested++; availableJobs--; anyJobsRequested = true; + + // Have we consumed all of our requests? + if ( availableJobs == 0 ) + { + break; + } } // if we did a pass and couldn't request any more jobs, then bail out From b4de4c42b44b57342fb582585d0d7d9cbdfdcaab Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 29 Apr 2023 11:47:35 +0930 Subject: [PATCH 019/129] Remove remnants of 32bits support for compiling FASTBuild - FASTBuild has been an exclusively 64bit process for sometime - The processes it can spawn are unaffected --- .../CoreTest/Tests/TestTCPConnectionPool.cpp | 2 +- Code/Core/Env/Types.h | 17 ++++------------- Code/Core/Process/Thread.cpp | 2 +- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp b/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp index 0a4d09d7d..ac672c59c 100644 --- a/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp +++ b/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp @@ -23,7 +23,7 @@ #define NUM_TEST_PASSES ( 16 ) // unique port for test in all configs so the tests can run in parallel -#ifdef WIN64 +#if defined( __WINDOWS__ ) #ifdef DEBUG #define TEST_PORT uint16_t( 21941 ) // arbitrarily chosen #else diff --git a/Code/Core/Env/Types.h b/Code/Core/Env/Types.h index 19c8a3445..09775fa22 100644 --- a/Code/Core/Env/Types.h +++ b/Code/Core/Env/Types.h @@ -72,28 +72,19 @@ typedef signed int int32_t; #endif #ifndef intptr_t - #if defined( WIN64 ) + #if defined( __WINDOWS__ ) typedef int64_t intptr_t; typedef uint64_t uintptr_t; - #elif defined( WIN32 ) - typedef int32_t intptr_t; - typedef uint32_t uintptr_t; #endif #endif #ifndef uintptr_t #if defined( __LINUX__ ) - #if defined( __X64__ ) || defined( __ARM64__ ) - typedef uint64_t uintptr_t; - #else - typedef uint32_t uintptr_t; - #endif + typedef uint64_t uintptr_t; #endif #endif #ifndef size_t - #if defined( WIN64 ) + #if defined( __WINDOWS__ ) typedef uint64_t size_t; - #elif defined( WIN32 ) - typedef uint32_t size_t; #endif #endif @@ -120,7 +111,7 @@ typedef signed int int32_t; // Warning disabling //------------------------------------------------------------------------------ -#if defined( WIN32 ) || defined( WIN64 ) +#if defined( __WINDOWS__ ) #define PRAGMA_DISABLE_PUSH_MSVC( num ) __pragma(warning(push)) \ __pragma(warning(disable:num)) #define PRAGMA_DISABLE_POP_MSVC __pragma(warning(pop)) diff --git a/Code/Core/Process/Thread.cpp b/Code/Core/Process/Thread.cpp index c35da8425..76af89591 100644 --- a/Code/Core/Process/Thread.cpp +++ b/Code/Core/Process/Thread.cpp @@ -213,7 +213,7 @@ bool Thread::IsRunning() const { PROFILE_FUNCTION; - #if defined( WIN32 ) || defined( WIN64 ) + #if defined( __WINDOWS__ ) ::Sleep( ms ); #elif defined( __APPLE__ ) || defined( __LINUX__ ) usleep( ms * 1000 ); From 326aa7d1fa5ea90425de7131d961d8cf7320c104 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 29 Apr 2023 12:22:40 +0930 Subject: [PATCH 020/129] Tidy up architecture defines - use compile built-ins for detection of X64 vs ARM instead of setting those in the bff files - this is necessary to allow multiple architectures to be generated in one compiler invocation for OSX Universal Binaries --- Code/Core/CoreTest/Tests/TestTimer.cpp | 2 +- Code/Core/Time/Timer.cpp | 4 ++-- Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp | 4 ++-- External/SDK/Clang/Linux/Clang10.bff | 1 - External/SDK/Clang/Linux/Clang3.bff | 1 - External/SDK/Clang/Linux/Clang6.bff | 1 - External/SDK/Clang/Linux/Clang_CI.bff | 1 - External/SDK/Clang/OSX/Clang12.bff | 2 -- External/SDK/Clang/OSX/Clang8.bff | 1 - External/SDK/Clang/OSX/Clang_CI.bff | 2 -- External/SDK/GCC/Linux/GCC4.bff | 1 - External/SDK/GCC/Linux/GCC7.bff | 1 - External/SDK/GCC/Linux/GCC9.bff | 1 - External/SDK/GCC/Linux/GCC_CI.bff | 1 - External/SDK/Windows/Windows10SDK.bff | 2 -- 15 files changed, 5 insertions(+), 20 deletions(-) diff --git a/Code/Core/CoreTest/Tests/TestTimer.cpp b/Code/Core/CoreTest/Tests/TestTimer.cpp index bca07f67c..272ca8461 100644 --- a/Code/Core/CoreTest/Tests/TestTimer.cpp +++ b/Code/Core/CoreTest/Tests/TestTimer.cpp @@ -32,7 +32,7 @@ void TestTimer::Validate() const Timer t; t.Start(); const int64_t before = t.GetNow(); - #if defined( __OSX__ ) && defined( __ARM64__ ) + #if defined( __OSX__ ) && defined( __aarch64__ ) // ARM // TODO:B Figure out why sleep granularity is so poor on Apple Silicon Thread::Sleep( 100 ); // sleep for 100ms #else diff --git a/Code/Core/Time/Timer.cpp b/Code/Core/Time/Timer.cpp index d5c4a4f1c..557d3363c 100644 --- a/Code/Core/Time/Timer.cpp +++ b/Code/Core/Time/Timer.cpp @@ -38,7 +38,7 @@ class GlobalTimerFrequencyInitializer Timer::s_Frequency = freq.QuadPart; #endif #if defined( __APPLE__ ) - #if defined( __ARM64__ ) + #if defined( __aarch64__ ) // ARM Timer::s_Frequency = 1000000000; #else mach_timebase_info_data_t info; @@ -64,7 +64,7 @@ int64_t Timer::GetNow() VERIFY( QueryPerformanceCounter( &now ) ); return now.QuadPart; #elif defined( __APPLE__ ) - #if defined( __ARM64__ ) + #if defined( __aarch64__ ) // ARM // mach_absolute_time seems to return the wrong time on Apple Silicon return (int64_t)clock_gettime_nsec_np( CLOCK_MONOTONIC ); #else diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp index 076f2664e..4c786348d 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp @@ -50,13 +50,13 @@ bool BFFMacros::IsDefined( const AString & token ) const return true; } #endif - #if defined( __X64__ ) + #if defined( __x86_64__ ) || defined( _M_X64 ) // X64 if ( token == "__X64__" ) { return true; } #endif - #if defined( __ARM64__ ) + #if defined( __aarch64__ ) || defined( _M_ARM64 ) // ARM if ( token == "__ARM64__" ) { return true; diff --git a/External/SDK/Clang/Linux/Clang10.bff b/External/SDK/Clang/Linux/Clang10.bff index be4ea90c7..fded8e6e3 100644 --- a/External/SDK/Clang/Linux/Clang10.bff +++ b/External/SDK/Clang/Linux/Clang10.bff @@ -31,7 +31,6 @@ Compiler( 'Compiler-Clang10' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/Linux/Clang3.bff b/External/SDK/Clang/Linux/Clang3.bff index 1d75392f9..efe3a08c7 100644 --- a/External/SDK/Clang/Linux/Clang3.bff +++ b/External/SDK/Clang/Linux/Clang3.bff @@ -28,7 +28,6 @@ Compiler( 'Compiler-Clang3' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/Linux/Clang6.bff b/External/SDK/Clang/Linux/Clang6.bff index 23eb30d9f..d55d81619 100644 --- a/External/SDK/Clang/Linux/Clang6.bff +++ b/External/SDK/Clang/Linux/Clang6.bff @@ -31,7 +31,6 @@ Compiler( 'Compiler-Clang6' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/Linux/Clang_CI.bff b/External/SDK/Clang/Linux/Clang_CI.bff index 926af79ab..971ea125f 100644 --- a/External/SDK/Clang/Linux/Clang_CI.bff +++ b/External/SDK/Clang/Linux/Clang_CI.bff @@ -30,7 +30,6 @@ Compiler( 'Compiler-Clang' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/OSX/Clang12.bff b/External/SDK/Clang/OSX/Clang12.bff index 5a042c8b4..b45210c99 100644 --- a/External/SDK/Clang/OSX/Clang12.bff +++ b/External/SDK/Clang/OSX/Clang12.bff @@ -34,7 +34,6 @@ Compiler( 'Compiler-Clang12' ) + ' -arch x86_64' // Intel 64-bit + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' @@ -80,7 +79,6 @@ Compiler( 'Compiler-Clang12' ) + ' -arch arm64' // Apple Silicon + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define - + ' -D__ARM64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/OSX/Clang8.bff b/External/SDK/Clang/OSX/Clang8.bff index bed1ff306..c5ea4497d 100644 --- a/External/SDK/Clang/OSX/Clang8.bff +++ b/External/SDK/Clang/OSX/Clang8.bff @@ -32,7 +32,6 @@ Compiler( 'Compiler-Clang8' ) + ' -m64' // x86_64 + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define - + ' -D__X64__' // Architecture define + ' -mmacosx-version-min=10.7' + ' -stdlib=libc++' diff --git a/External/SDK/Clang/OSX/Clang_CI.bff b/External/SDK/Clang/OSX/Clang_CI.bff index d987a23c4..6a52dcd7c 100644 --- a/External/SDK/Clang/OSX/Clang_CI.bff +++ b/External/SDK/Clang/OSX/Clang_CI.bff @@ -32,7 +32,6 @@ Compiler( 'Compiler-Clang12' ) + ' -arch x86_64' // Intel 64-bit + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' @@ -78,7 +77,6 @@ Compiler( 'Compiler-Clang12' ) + ' -arch arm64' // Apple Silicon + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define - + ' -D__ARM64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/GCC/Linux/GCC4.bff b/External/SDK/GCC/Linux/GCC4.bff index a271f15c8..bdcb3c9a3 100644 --- a/External/SDK/GCC/Linux/GCC4.bff +++ b/External/SDK/GCC/Linux/GCC4.bff @@ -33,7 +33,6 @@ Compiler( 'Compiler-GCC4' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h // Include paths diff --git a/External/SDK/GCC/Linux/GCC7.bff b/External/SDK/GCC/Linux/GCC7.bff index e1ea0a313..01628903a 100644 --- a/External/SDK/GCC/Linux/GCC7.bff +++ b/External/SDK/GCC/Linux/GCC7.bff @@ -33,7 +33,6 @@ Compiler( 'Compiler-GCC7' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h // Include paths diff --git a/External/SDK/GCC/Linux/GCC9.bff b/External/SDK/GCC/Linux/GCC9.bff index b42eabf25..677df0531 100644 --- a/External/SDK/GCC/Linux/GCC9.bff +++ b/External/SDK/GCC/Linux/GCC9.bff @@ -33,7 +33,6 @@ Compiler( 'Compiler-GCC9' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h // Include paths diff --git a/External/SDK/GCC/Linux/GCC_CI.bff b/External/SDK/GCC/Linux/GCC_CI.bff index 511e99cc9..7168509b1 100644 --- a/External/SDK/GCC/Linux/GCC_CI.bff +++ b/External/SDK/GCC/Linux/GCC_CI.bff @@ -32,7 +32,6 @@ Compiler( 'Compiler-GCC' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h // Include paths diff --git a/External/SDK/Windows/Windows10SDK.bff b/External/SDK/Windows/Windows10SDK.bff index 1135b040d..9e987ece1 100644 --- a/External/SDK/Windows/Windows10SDK.bff +++ b/External/SDK/Windows/Windows10SDK.bff @@ -61,7 +61,6 @@ .CommonCompilerOptions = .WindowsSDK_IncludePaths + .WindowsSDK_Defines + ' -DWIN64' - + ' -D__X64__' .CompilerOptions = .CommonCompilerOptions .CompilerOptionsC = .CommonCompilerOptions @@ -85,7 +84,6 @@ .CommonCompilerOptions = .WindowsSDK_IncludePaths + .WindowsSDK_Defines + ' -DWIN64' - + ' -D__X64__' .CompilerOptions = .CommonCompilerOptions .CompilerOptionsC = .CommonCompilerOptions From f9504642d8dc015a3f56d0713adb22dba2f053ec Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 29 Apr 2023 12:39:21 +0930 Subject: [PATCH 021/129] OSX: Use Clang12 to build FASTBuild by default --- External/SDK/Clang/Clang.bff | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/External/SDK/Clang/Clang.bff b/External/SDK/Clang/Clang.bff index 025ca4615..3cc75c65d 100644 --- a/External/SDK/Clang/Clang.bff +++ b/External/SDK/Clang/Clang.bff @@ -9,8 +9,8 @@ #define USING_CLANG_10 #endif #if __OSX__ && !CI_BUILD - #define USING_CLANG_8 - //#define USING_CLANG_12 + //#define USING_CLANG_8 + #define USING_CLANG_12 #endif #if __WINDOWS__ && !CI_BUILD //#define USING_CLANG_7 From 5fdc9fa6c149a14c5337f43e06dbaec1ad1828af Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 10:57:41 +0930 Subject: [PATCH 022/129] OSX: Simplify Clang12 configs - de-duplicate config common to x64 and ARM --- External/SDK/Clang/OSX/Clang12.bff | 65 +++++++++-------------------- External/SDK/Clang/OSX/Clang_CI.bff | 64 +++++++++------------------- 2 files changed, 38 insertions(+), 91 deletions(-) diff --git a/External/SDK/Clang/OSX/Clang12.bff b/External/SDK/Clang/OSX/Clang12.bff index b45210c99..6f3770dde 100644 --- a/External/SDK/Clang/OSX/Clang12.bff +++ b/External/SDK/Clang/OSX/Clang12.bff @@ -22,16 +22,13 @@ Compiler( 'Compiler-Clang12' ) // ToolChain //------------------------------------------------------------------------------ -.ToolChain_Clang_OSX = +.ToolChain_Clang_OSXCommon = [ - .Platform = 'x64OSX' - // Compiler Options .Compiler = 'Compiler-Clang12' .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output + ' -c' // Compile only + ' -g' // Generate debug info - + ' -arch x86_64' // Intel 64-bit + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define @@ -47,7 +44,6 @@ Compiler( 'Compiler-Clang12' ) + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types + ' -Wno-implicit-exception-spec-mismatch' // Fires on our new/delete operator (Clang bug?) - .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' .CompilerOptionsC = ' -x c $CommonCompilerOptions$' @@ -57,7 +53,7 @@ Compiler( 'Compiler-Clang12' ) // Linker .Linker = '$Clang12_BasePath$/clang++' - .LinkerOptions = '"%1" -o "%2" -g -arch x86_64' + .LinkerOptions = '"%1" -o "%2" -g' // File Extensions .LibExtension = '.a' @@ -67,49 +63,26 @@ Compiler( 'Compiler-Clang12' ) .UseExceptions = ' -fexceptions' ] +.ToolChain_Clang_OSX = +[ + Using( .ToolChain_Clang_OSXCommon ) + + // Intel 64-bit + .Platform = 'x64OSX' + .CompilerOptions + ' -arch x86_64' + .CompilerOptionsC + ' -arch x86_64' + .LinkerOptions + ' -arch x86_64' +] + .ToolChain_Clang_ARMOSX = [ + Using( .ToolChain_Clang_OSXCommon ) + + // Apple ARM Silicon .Platform = 'ARMOSX' - - // Compiler Options - .Compiler = 'Compiler-Clang12' - .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output - + ' -c' // Compile only - + ' -g' // Generate debug info - + ' -arch arm64' // Apple Silicon - + ' -D__OSX__' // Platform define - + ' -D__APPLE__' // Platform define - - // Include paths - + ' -I./' - - // Enable warnings - + ' -Wall -Werror -Wfatal-errors' // warnings as errors - + ' -Wextra' - - // Disabled warnings - + ' -Wno-#pragma-messages' - + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types - + ' -Wno-implicit-exception-spec-mismatch' // Fires on our new/delete operator (Clang bug?) - - - .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' - .CompilerOptionsC = ' -x c $CommonCompilerOptions$' - - // Librarian - .Librarian = '$Clang12_BasePath$/ar' - .LibrarianOptions = 'rcs "%2" "%1"' - - // Linker - .Linker = '$Clang12_BasePath$/clang++' - .LinkerOptions = '"%1" -o "%2" -g -arch arm64' - - // File Extensions - .LibExtension = '.a' - .ExeExtension = '' - - // Exception Control - .UseExceptions = ' -fexceptions' + .CompilerOptions + ' -arch arm64' + .CompilerOptionsC + ' -arch arm64' + .LinkerOptions + ' -arch arm64' ] //------------------------------------------------------------------------------ diff --git a/External/SDK/Clang/OSX/Clang_CI.bff b/External/SDK/Clang/OSX/Clang_CI.bff index 6a52dcd7c..747c87f22 100644 --- a/External/SDK/Clang/OSX/Clang_CI.bff +++ b/External/SDK/Clang/OSX/Clang_CI.bff @@ -20,16 +20,13 @@ Compiler( 'Compiler-Clang12' ) // ToolChain //------------------------------------------------------------------------------ -.ToolChain_Clang_OSX = +.ToolChain_Clang_OSXCommon = [ - .Platform = 'x64OSX' - // Compiler Options .Compiler = 'Compiler-Clang12' .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output + ' -c' // Compile only + ' -g' // Generate debug info - + ' -arch x86_64' // Intel 64-bit + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define @@ -55,7 +52,7 @@ Compiler( 'Compiler-Clang12' ) // Linker .Linker = 'CLANGXX_BINARY' - .LinkerOptions = '"%1" -o "%2" -g -arch x86_64' + .LinkerOptions = '"%1" -o "%2" -g' // File Extensions .LibExtension = '.a' @@ -65,49 +62,26 @@ Compiler( 'Compiler-Clang12' ) .UseExceptions = ' -fexceptions' ] +.ToolChain_Clang_OSX = +[ + Using( .ToolChain_Clang_OSXCommon ) + + // Intel 64-bit + .Platform = 'x64OSX' + .CompilerOptions + ' -arch x86_64' + .CompilerOptionsC + ' -arch x86_64' + .LinkerOptions + ' -arch x86_64' +] + .ToolChain_Clang_ARMOSX = [ + Using( .ToolChain_Clang_OSXCommon ) + + // Apple ARM Silicon .Platform = 'ARMOSX' - - // Compiler Options - .Compiler = 'Compiler-Clang12' - .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output - + ' -c' // Compile only - + ' -g' // Generate debug info - + ' -arch arm64' // Apple Silicon - + ' -D__OSX__' // Platform define - + ' -D__APPLE__' // Platform define - - // Include paths - + ' -I./' - - // Enable warnings - + ' -Wall -Werror -Wfatal-errors' // warnings as errors - + ' -Wextra' - - // Disabled warnings - + ' -Wno-#pragma-messages' - + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types - + ' -Wno-implicit-exception-spec-mismatch' // Fires on our new/delete operator (Clang bug?) - - - .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' - .CompilerOptionsC = ' -x c $CommonCompilerOptions$' - - // Librarian - .Librarian = '/usr/bin/ar' - .LibrarianOptions = 'rcs "%2" "%1"' - - // Linker - .Linker = 'CLANGXX_BINARY' - .LinkerOptions = '"%1" -o "%2" -g -arch arm64' - - // File Extensions - .LibExtension = '.a' - .ExeExtension = '' - - // Exception Control - .UseExceptions = ' -fexceptions' + .CompilerOptions + ' -arch arm64' + .CompilerOptionsC + ' -arch arm64' + .LinkerOptions + ' -arch arm64' ] //------------------------------------------------------------------------------ From 7a5221053a21b149a3e70eaf3e0241c5d475245e Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 11:06:37 +0930 Subject: [PATCH 023/129] [Improvement][OSX] Add native Apple Silicon support using Universal binaries - combine existing arm and x64 executables together into universal binaries via new "OSX" targets - gated on availability of Clang12 or newer --- Code/.FASTBuild/HelperFunctions.bff | 37 ++++++++++++++-- Code/Tools/FBuild/FBuild/FBuild.bff | 13 ++++++ .../FBuild/FBuildWorker/FBuildWorker.bff | 15 ++++++- Code/fbuild.bff | 44 +++++++++++++++---- 4 files changed, 97 insertions(+), 12 deletions(-) diff --git a/Code/.FASTBuild/HelperFunctions.bff b/Code/.FASTBuild/HelperFunctions.bff index f74179a2e..62102fd1d 100644 --- a/Code/.FASTBuild/HelperFunctions.bff +++ b/Code/.FASTBuild/HelperFunctions.bff @@ -59,9 +59,16 @@ function CreateCommonAliases( .ProjectName ) Alias( '$ProjectName$-TSan' ) { .Targets = { '$ProjectName$-X64Linux-TSan', '$ProjectName$-X64ClangLinux-TSan' } } #endif #if __OSX__ - Alias( '$ProjectName$-Debug' ) { .Targets = { '$ProjectName$-X64OSX-Debug' } } - Alias( '$ProjectName$-Profile' ) { .Targets = { '$ProjectName$-X64OSX-Profile' } } - Alias( '$ProjectName$-Release' ) { .Targets = { '$ProjectName$-X64OSX-Release' } } + #if CLANG_SUPPORTS_ARMOSX + // Universal Binaries + Alias( '$ProjectName$-Debug' ) { .Targets = { '$ProjectName$-OSX-Debug' } } + Alias( '$ProjectName$-Profile' ) { .Targets = { '$ProjectName$-OSX-Profile' } } + Alias( '$ProjectName$-Release' ) { .Targets = { '$ProjectName$-OSX-Release' } } + #else + Alias( '$ProjectName$-Debug' ) { .Targets = { '$ProjectName$-X64OSX-Debug' } } + Alias( '$ProjectName$-Profile' ) { .Targets = { '$ProjectName$-X64OSX-Profile' } } + Alias( '$ProjectName$-Release' ) { .Targets = { '$ProjectName$-X64OSX-Release' } } + #endif #endif // Per-Platform @@ -70,6 +77,10 @@ function CreateCommonAliases( .ProjectName ) Alias( '$ProjectName$-X64Linux' ) { .Targets = { '$ProjectName$-X64Linux-Debug', '$ProjectName$-X64Linux-Release', '$ProjectName$-X64Linux-Profile', '$ProjectName$-X64Linux-ASan', '$ProjectName$-X64Linux-TSan' } } Alias( '$ProjectName$-X64ClangLinux' ) { .Targets = { '$ProjectName$-X64ClangLinux-Debug', '$ProjectName$-X64ClangLinux-Release', '$ProjectName$-X64ClangLinux-Profile', '$ProjectName$-X64ClangLinux-ASan', '$ProjectName$-X64ClangLinux-MSan', '$ProjectName$-X64ClangLinux-TSan' } } Alias( '$ProjectName$-X64OSX' ) { .Targets = { '$ProjectName$-X64OSX-Debug', '$ProjectName$-X64OSX-Release', '$ProjectName$-X64OSX-Profile' } } + #if CLANG_SUPPORTS_ARMOSX + Alias( '$ProjectName$-ARMOSX' ) { .Targets = { '$ProjectName$-ARMOSX-Debug', '$ProjectName$-ARMOSX-Release', '$ProjectName$-ARMOSX-Profile' } } + Alias( '$ProjectName$-OSX' ) { .Targets = { '$ProjectName$-OSX-Debug', '$ProjectName$-OSX-Release', '$ProjectName$-OSX-Profile' } } + #endif // All Alias( '$ProjectName$' ) @@ -159,4 +170,24 @@ function CreateVCXProject_Exe( .Name CreateVCXProject_Lib( .Name, .Path, .Configs, .ExtraOptions ) } + +// CreateUniversalBinary +//------------------------------------------------------------------------------ +#if CLANG_SUPPORTS_ARMOSX + function CreateUniversalBinary( .ProjectName + .ProjectPath + .BuildConfigName + .OutputBase ) + { + Exec( '$ProjectName$-Exe-OSX-$BuildConfigName$' ) + { + .ExecExecutable = '/usr/bin/lipo' + .ExecInput = { '$ProjectName$-Exe-x64OSX-$BuildConfigName$', '$ProjectName$-Exe-ARMOSX-$BuildConfigName$' } + .ExecOutput = '$OutputBase$/OSX-$BuildConfigName$/$ProjectPath$/$ProjectName$' + .ExecArguments = '-create -output %2 %1' + } + Alias( '$ProjectName$-OSX-$BuildConfigName$' ) { .Targets = '$ProjectName$-Exe-OSX-$BuildConfigName$' } + } +#endif + //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuild/FBuild.bff b/Code/Tools/FBuild/FBuild/FBuild.bff index 8f4d31401..f3c260a6c 100644 --- a/Code/Tools/FBuild/FBuild/FBuild.bff +++ b/Code/Tools/FBuild/FBuild/FBuild.bff @@ -94,6 +94,19 @@ #endif } + // Create Universal binaries + #if CLANG_SUPPORTS_ARMOSX + ForEach( .BuildConfig in .BuildConfigs ) + { + Using( .BuildConfig ) + If( .Platform == 'ARMOSX' ) + { + CreateUniversalBinary( .ProjectName, .ProjectPath, .BuildConfigName, .OutputBase ) + ^'Targets_OSX_$BuildConfigName$' + { '$ProjectName$-OSX-$BuildConfigName$' } + } + } + #endif + // Aliases //-------------------------------------------------------------------------- CreateCommonAliases( .ProjectName ) diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff index 37010caf3..d7e37a276 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff @@ -135,7 +135,20 @@ ^ProjectConfigs + .ProjectConfig #endif } - + + // Create Universal binaries + #if CLANG_SUPPORTS_ARMOSX + ForEach( .BuildConfig in .BuildConfigs ) + { + Using( .BuildConfig ) + If( .Platform == 'ARMOSX' ) + { + CreateUniversalBinary( .ProjectName, .ProjectPath, .BuildConfigName, .OutputBase ) + ^'Targets_OSX_$BuildConfigName$' + { '$ProjectName$-OSX-$BuildConfigName$' } + } + } + #endif + // Aliases //-------------------------------------------------------------------------- CreateCommonAliases( .ProjectName ) diff --git a/Code/fbuild.bff b/Code/fbuild.bff index 4b98093a2..d29eef84e 100644 --- a/Code/fbuild.bff +++ b/Code/fbuild.bff @@ -388,9 +388,14 @@ Settings .Targets_x64OSX_Debug = {} .Targets_x64OSX_Profile = {} .Targets_x64OSX_Release = {} -.Targets_ARMOSX_Debug = {} -.Targets_ARMOSX_Profile = {} -.Targets_ARMOSX_Release = {} +#if CLANG_SUPPORTS_ARMOSX + .Targets_ARMOSX_Debug = {} + .Targets_ARMOSX_Profile = {} + .Targets_ARMOSX_Release = {} + .Targets_OSX_Debug = {} + .Targets_OSX_Profile = {} + .Targets_OSX_Release = {} +#endif // External #include "..\External\LZ4\LZ4.bff" @@ -420,6 +425,14 @@ ForEach( .BuildConfig in .BuildConfigs ) Using( .BuildConfig ) Alias( 'All-$Platform$-$BuildConfigName$' ) { .Targets = .'Targets_$Platform$_$BuildConfigName$' } + + // Create additional Universal targets + #if CLANG_SUPPORTS_ARMOSX + If( .Platform == 'ARMOSX' ) + { + Alias( 'All-OSX-$BuildConfigName$' ) { .Targets = .'Targets_OSX_$BuildConfigName$' } + } + #endif } // Exes @@ -432,7 +445,18 @@ Alias( 'Exes' ) // Aliases : All-$Platform$ //------------------------------------------------------------------------------ -.Platforms = { 'x64', 'x64Clang', 'x64Linux', 'x64ClangLinux', 'x64OSX', 'ARMOSX' } +.Platforms = +{ + 'x64', + 'x64Clang', + 'x64Linux', + 'x64ClangLinux', + 'x64OSX' + #if CLANG_SUPPORTS_ARMOSX + 'ARMOSX' + 'OSX' + #endif +} .PlatformConfigs_x64 = { 'Debug', 'Analyze', 'Profile', 'Release', 'ASan', 'TSan' } .PlatformConfigs_x64Clang = { 'Debug', 'Analyze', 'Profile', 'Release', 'ASan', 'TSan' } .PlatformConfigs_x64Linux = { 'Debug', 'Profile', 'Release', 'ASan', 'TSan' } @@ -441,7 +465,10 @@ Alias( 'Exes' ) + { 'ASan', 'MSan', 'TSan' } #endif .PlatformConfigs_x64OSX = { 'Debug', 'Profile', 'Release' } -.PlatformConfigs_ARMOSX = { 'Debug', 'Profile', 'Release' } +#if CLANG_SUPPORTS_ARMOSX + .PlatformConfigs_ARMOSX = { 'Debug', 'Profile', 'Release' } + .PlatformConfigs_OSX = { 'Debug', 'Profile', 'Release' } +#endif ForEach( .Platform in .Platforms ) { Alias( 'All-$Platform$' ) @@ -480,9 +507,10 @@ ForEach( .Platform in .Platforms ) #if __OSX__ Alias( 'All' ) { - .Targets = { 'All-x64OSX' } - #if CLANG_SUPPORTS_OSXARM - + { 'All-ARMOSX' } + #if CLANG_SUPPORTS_ARMOSX + .Targets = { 'All-OSX' } // X64, ARM & Universal + #else + .Targets = { 'All-x64OSX' } // X64 only #endif } #endif From e6487e754df4878d91b6c578b965a4371054e538 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 11:24:31 +0930 Subject: [PATCH 024/129] git CI uses Universal binary targets for OSX --- .github/workflows/OSX.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/OSX.yml b/.github/workflows/OSX.yml index 5c6b2d3a9..854321d42 100644 --- a/.github/workflows/OSX.yml +++ b/.github/workflows/OSX.yml @@ -43,7 +43,7 @@ jobs: - name: Build if: ${{ startsWith(matrix.cfg, 'Build') }} - run: ${FBUILD_PATH} -nostoponerror -summary All-x64OSX-{Debug,Profile,Release} + run: ${FBUILD_PATH} -nostoponerror -summary All-OSX-{Debug,Profile,Release} - name: Tests # -j1 on CI nodes avoids timeouts (CI nodes have only 2 cores) @@ -52,4 +52,4 @@ jobs: - name: Build (NoUnity) if: ${{ startsWith(matrix.cfg, 'Build') }} - run: ${FBUILD_PATH} -nostoponerror -summary -nounity -clean All-x64OSX-Debug + run: ${FBUILD_PATH} -nostoponerror -summary -nounity -clean All-OSX-Debug From 5e15f13253a69e7a7483d030a4340cb2ad3c6d05 Mon Sep 17 00:00:00 2001 From: Dandielo Date: Sun, 30 Apr 2023 11:30:03 +0900 Subject: [PATCH 025/129] Fix collision errors on chained ObjectLists (#966) * Chained ObjectLists now properly handle sub-directories. * Fix test by increasing number of objects seend and built. --- Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp | 4 ++-- Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h | 1 + .../Data/TestObjectList/ObjectListChaining/A/file.1.cpp | 0 Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp | 6 +++--- 4 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 Code/Tools/FBuild/FBuildTest/Data/TestObjectList/ObjectListChaining/A/file.1.cpp diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index a5b760176..b28e32209 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -467,7 +467,7 @@ ObjectListNode::~ObjectListNode() = default; } // create the object that will compile the above file - if ( CreateDynamicObjectNode( nodeGraph, n->GetName(), AString::GetEmpty() ) == false ) + if ( CreateDynamicObjectNode( nodeGraph, n->GetName(), objListNode->GetCompilerOutputPath() ) == false ) { return false; // CreateDynamicObjectNode will have emitted error } @@ -837,7 +837,7 @@ void ObjectListNode::EnumerateInputFiles( void (*callback)( const AString & inpu { callback( file, AString::GetEmpty(), userData ); } - + // Dynamically discovered files for ( size_t i = m_ObjectListInputStartIndex; i < m_ObjectListInputEndIndex; ++i ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h index 992fb7fb9..afce005ea 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h @@ -38,6 +38,7 @@ class ObjectListNode : public Node void GetInputFiles( Args & fullArgs, const AString & pre, const AString & post, bool objectsInsteadOfLibs ) const; void GetInputFiles( Array< AString > & files ) const; + inline const AString & GetCompilerOutputPath() const { return m_CompilerOutputPath; } inline const AString & GetCompilerOptions() const { return m_CompilerOptions; } inline const AString & GetCompiler() const { return m_Compiler; } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/ObjectListChaining/A/file.1.cpp b/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/ObjectListChaining/A/file.1.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp index ad211b3ae..ef9760769 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp @@ -183,7 +183,7 @@ void TestObjectList::ObjectListChaining() const // Check stats // Seen, Built, Type CheckStatsNode( 2, 2, Node::OBJECT_LIST_NODE ); - CheckStatsNode( 2, 2, Node::OBJECT_NODE ); + CheckStatsNode( 4, 4, Node::OBJECT_NODE ); CheckStatsNode( 1, 1, Node::DIRECTORY_LIST_NODE ); fBuild.SerializeDepGraphToText( "ObjectList2", depGraphText1 ); @@ -198,7 +198,7 @@ void TestObjectList::ObjectListChaining() const // Check stats // Seen, Built, Type CheckStatsNode( 2, 0, Node::OBJECT_LIST_NODE ); - CheckStatsNode( 2, 0, Node::OBJECT_NODE ); + CheckStatsNode( 4, 0, Node::OBJECT_NODE ); CheckStatsNode( 1, 1, Node::DIRECTORY_LIST_NODE ); } @@ -213,7 +213,7 @@ void TestObjectList::ObjectListChaining() const // Check stats // Seen, Built, Type CheckStatsNode( 2, 0, Node::OBJECT_LIST_NODE ); - CheckStatsNode( 2, 0, Node::OBJECT_NODE ); + CheckStatsNode( 4, 0, Node::OBJECT_NODE ); CheckStatsNode( 1, 1, Node::DIRECTORY_LIST_NODE ); fBuild.SerializeDepGraphToText( "ObjectList2", depGraphText2 ); From 2f7bf468581efa65c63428f15651d5dbf23d3d9e Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 12:42:07 +0930 Subject: [PATCH 026/129] Post-intergation: Chained ObjectList fix --- Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index 2b5083001..59880707d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -63,7 +63,7 @@ class NodeGraphHeader } inline ~NodeGraphHeader() = default; - enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 169 }; + enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 170 }; bool IsValid() const; bool IsCompatibleVersion() const { return m_Version == NODE_GRAPH_CURRENT_VERSION; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index b28e32209..814299cd1 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -837,7 +837,7 @@ void ObjectListNode::EnumerateInputFiles( void (*callback)( const AString & inpu { callback( file, AString::GetEmpty(), userData ); } - + // Dynamically discovered files for ( size_t i = m_ObjectListInputStartIndex; i < m_ObjectListInputEndIndex; ++i ) { From 0348efbc287e228b4fa690755b679a3000bd9f67 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 14:30:18 +0930 Subject: [PATCH 027/129] [Improvement] FBuildWorker will restart every 4 hours to improve reliability - this helps ensure intermittent issues don't impact overall worker pool reliability --- Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp | 15 ++++++++++++--- Code/Tools/FBuild/FBuildWorker/Worker/Worker.h | 3 ++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp index a90f561ed..f717cf1b5 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp @@ -207,7 +207,7 @@ uint32_t Worker::WorkThread() UpdateUI(); - CheckForExeUpdate(); + CheckIfRestartNeeded(); PROFILE_SYNCHRONIZE @@ -432,9 +432,9 @@ void Worker::UpdateUI() m_UIUpdateTimer.Start(); } -// CheckForExeUpdate +// CheckIfRestartNeeded //------------------------------------------------------------------------------ -void Worker::CheckForExeUpdate() +void Worker::CheckIfRestartNeeded() { PROFILE_FUNCTION; @@ -455,6 +455,15 @@ void Worker::CheckForExeUpdate() return; // not running as a copy to allow restarts } + // Check if periodic restart time has been reached + const float periodicRestartSecs = ( 4.0f * 60.0f * 60.0f ); // 4 hours + if ( m_PeriodicRestartTimer.GetElapsed() > periodicRestartSecs ) + { + m_RestartNeeded = true; + JobQueueRemote::Get().SignalStopWorkers(); + return; + } + // get the current last write time const uint64_t lastWriteTime = FileIO::GetFileLastWriteTime( m_BaseExeName ); diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h index 076817b45..731cb2888 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h @@ -40,7 +40,7 @@ class Worker : public Singleton void UpdateAvailability(); void UpdateUI(); - void CheckForExeUpdate(); + void CheckIfRestartNeeded(); bool HasEnoughDiskSpace(); bool HasEnoughMemory(); @@ -61,6 +61,7 @@ class Worker : public Singleton uint64_t m_LastWriteTime; bool m_WantToQuit; bool m_RestartNeeded; + Timer m_PeriodicRestartTimer; Timer m_UIUpdateTimer; FileStream m_TargetIncludeFolderLock; #if defined( __WINDOWS__ ) From 51ff53bed76f300779cdbb9e6bdbb101b50ab0a8 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 14:48:58 +0930 Subject: [PATCH 028/129] [Improvment] Remove some unecessary lock contention in FBuildWorker - avoid locking the worker list unnecessarily when not accepting work --- .../Tools/FBuild/FBuildCore/Protocol/Server.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index 6c3552ac4..9f6701346 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -614,18 +614,19 @@ void Server::FindNeedyClients() } PROFILE_FUNCTION; + + // determine job availability + int32_t availableJobs = (int32_t)WorkerThreadRemote::GetNumCPUsToUse(); + if ( availableJobs == 0 ) + { + return; + } + ++availableJobs; // over request to parallelize building/network transfers + { MutexHolder mh( m_ClientListMutex ); - // determine job availability - int availableJobs = (int)WorkerThreadRemote::GetNumCPUsToUse(); - if ( availableJobs == 0 ) - { - return; - } - ++availableJobs; // over request to parallelize building/network transfers - for ( ClientState * cs : m_ClientList ) { MutexHolder mh2( cs->m_Mutex ); From 7557f790b19c50233d31f58a63cae6152c798db3 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 15:02:48 +0930 Subject: [PATCH 029/129] Mutex: Add TryMutexHolder to support RAII patterns with speculatively acquired Mutexes --- Code/Core/CoreTest/Tests/TestMutex.cpp | 25 ++++++++++++++++++----- Code/Core/Process/Mutex.h | 28 ++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/Code/Core/CoreTest/Tests/TestMutex.cpp b/Code/Core/CoreTest/Tests/TestMutex.cpp index 1d2bea5af..d1acfea9e 100644 --- a/Code/Core/CoreTest/Tests/TestMutex.cpp +++ b/Code/Core/CoreTest/Tests/TestMutex.cpp @@ -94,7 +94,7 @@ void TestMutex::TestMultiLockUnlock() const //------------------------------------------------------------------------------ void TestMutex::TryLock() const { - // Ensure lock can be acquired multiple times + // Ensure lock can be acquired Mutex m; TEST_ASSERT( m.TryLock() ); m.Unlock(); @@ -106,10 +106,25 @@ void TestMutex::TryLockMultiple() const { // Ensure lock can be acquired multiple times Mutex m; - TEST_ASSERT( m.TryLock() ); - TEST_ASSERT( m.TryLock() ); - m.Unlock(); - m.Unlock(); + { + // Manual locks + TEST_ASSERT( m.TryLock() ); + TEST_ASSERT( m.TryLock() ); + m.Unlock(); + m.Unlock(); + } + { + // RAII lock + MutexHolder mh( m ); + MutexHolder m2( m ); + } + { + // RAII lock (Try) + TryMutexHolder mh( m ); + TEST_ASSERT( mh.IsLocked() ); + TryMutexHolder mh2( m ); + TEST_ASSERT( mh2.IsLocked() ); + } } // TryLockMixed diff --git a/Code/Core/Process/Mutex.h b/Code/Core/Process/Mutex.h index 8a4597832..ceed04a59 100644 --- a/Code/Core/Process/Mutex.h +++ b/Code/Core/Process/Mutex.h @@ -52,4 +52,32 @@ class MutexHolder Mutex & m_Mutex; }; +// TryMutexHolder +//------------------------------------------------------------------------------ +class TryMutexHolder +{ +public: + explicit TryMutexHolder( Mutex & mutex ) + : m_Mutex( mutex ) + , m_Locked( mutex.TryLock() ) + { + } + ~TryMutexHolder() + { + if ( m_Locked ) + { + m_Mutex.Unlock(); + } + } + + [[nodiscard]] bool IsLocked() const { return m_Locked; } + +private: + TryMutexHolder( const TryMutexHolder & other ) = delete; + void operator = ( TryMutexHolder & other ) = delete; + + Mutex & m_Mutex; + const bool m_Locked; +}; + //------------------------------------------------------------------------------ From d055722eb0c99f46973e55d0c9688f0cc615dd0f Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 16:31:13 +0930 Subject: [PATCH 030/129] [Improvement] Eliminate various instances of lock contention in FBuildWorker --- .../FBuild/FBuildCore/Protocol/Server.cpp | 51 +++++++++++-------- .../Tools/FBuild/FBuildCore/Protocol/Server.h | 8 +-- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index 9f6701346..83b5ac561 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -290,7 +290,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgConn // take note of initial status of client ClientState * cs = (ClientState *)connection->GetUserData(); MutexHolder mh( cs->m_Mutex ); - cs->m_NumJobsAvailable = msg->GetNumJobsAvailable(); + cs->m_NumJobsAvailable.Store( msg->GetNumJobsAvailable() ); cs->m_ProtocolVersionMinor = msg->GetProtocolVersionMinor(); cs->m_HostName = msg->GetHostName(); } @@ -301,8 +301,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgStat { // take note of latest status of client ClientState * cs = (ClientState *)connection->GetUserData(); - MutexHolder mh( cs->m_Mutex ); - cs->m_NumJobsAvailable = msg->GetNumJobsAvailable(); + cs->m_NumJobsAvailable.Store( msg->GetNumJobsAvailable() ); // Wake main thread to request jobs JobQueueRemote::Get().WakeMainThread(); @@ -314,9 +313,8 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgNoJo { // We requested a job, but the client didn't have any left ClientState * cs = (ClientState *)connection->GetUserData(); - MutexHolder mh( cs->m_Mutex ); - ASSERT( cs->m_NumJobsRequested > 0 ); - cs->m_NumJobsRequested--; + ASSERT( cs->m_NumJobsRequested.Load() > 0 ); + cs->m_NumJobsRequested.Decrement(); } // Process( MsgJob ) @@ -325,11 +323,12 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgJob { ClientState * cs = (ClientState *)connection->GetUserData(); { + ASSERT( cs->m_NumJobsRequested.Load() > 0 ); + cs->m_NumJobsRequested.Decrement(); + cs->m_NumJobsActive.Increment(); + MutexHolder mh( cs->m_Mutex ); - ASSERT( cs->m_NumJobsRequested > 0 ); - cs->m_NumJobsRequested--; - cs->m_NumJobsActive++; - + // deserialize job ConstMemoryStream ms( payload, payloadSize ); @@ -627,12 +626,13 @@ void Server::FindNeedyClients() { MutexHolder mh( m_ClientListMutex ); - for ( ClientState * cs : m_ClientList ) + // determine if all available job slots are in use + for ( const ClientState * cs : m_ClientList ) { - MutexHolder mh2( cs->m_Mutex ); - // any jobs requested or in progress reduce the available count - const int32_t reservedJobs = (int32_t)( cs->m_NumJobsRequested + cs->m_NumJobsActive ); + const uint32_t jobsRequested = cs->m_NumJobsRequested.Load(); + const uint32_t jobsActive = cs->m_NumJobsActive.Load(); + const int32_t reservedJobs = static_cast( jobsRequested + jobsActive ); availableJobs -= reservedJobs; if ( availableJobs <= 0 ) { @@ -653,18 +653,24 @@ void Server::FindNeedyClients() for ( ClientState * cs : m_ClientList ) { - MutexHolder mh2( cs->m_Mutex ); - - const size_t reservedJobs = cs->m_NumJobsRequested; + const uint32_t reservedJobs = cs->m_NumJobsRequested.Load(); - if ( reservedJobs >= cs->m_NumJobsAvailable ) + if ( reservedJobs >= cs->m_NumJobsAvailable.Load() ) { continue; // we've maxed out the requests to this worker } // request job from this client - msg.Send( cs->m_Connection ); - cs->m_NumJobsRequested++; + { + // Acquire the lock but don't wait if unavailable + TryMutexHolder tryLock( cs->m_Mutex ); + if ( tryLock.IsLocked() == false ) + { + continue; // Skip this worker for now + } + msg.Send( cs->m_Connection ); + } + cs->m_NumJobsRequested.Increment(); availableJobs--; anyJobsRequested = true; @@ -719,9 +725,10 @@ void Server::FinalizeCompletedJobs() ms.WriteBuffer( job->GetData(), job->GetDataSize() ); { + ASSERT( cs->m_NumJobsActive.Load() > 0 ); + cs->m_NumJobsActive.Decrement(); + MutexHolder mh2( cs->m_Mutex ); - ASSERT( cs->m_NumJobsActive ); - cs->m_NumJobsActive--; if ( job->GetResultCompressionLevel() == 0 ) { diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h index a4d654395..8be313ed8 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h @@ -67,15 +67,15 @@ class Server : public TCPConnectionPool , m_WaitingJobs( 16, true ) {} - inline bool operator < ( const ClientState & other ) const { return ( m_NumJobsAvailable > other.m_NumJobsAvailable ); } + inline bool operator < ( const ClientState & other ) const { return ( m_NumJobsAvailable.Load() > other.m_NumJobsAvailable.Load() ); } Mutex m_Mutex; const Protocol::IMessage * m_CurrentMessage = nullptr; const ConnectionInfo * m_Connection = nullptr; - uint32_t m_NumJobsAvailable = 0; - uint32_t m_NumJobsRequested = 0; - uint32_t m_NumJobsActive = 0; + Atomic m_NumJobsAvailable; + Atomic m_NumJobsRequested; + Atomic m_NumJobsActive; uint8_t m_ProtocolVersionMinor = 0; AString m_HostName; From c78009ad8b2a3c11427735dbe7cf912521f9675f Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 30 Apr 2023 17:37:17 +0930 Subject: [PATCH 031/129] Fix requested job count becoming negative under heavy load - result of request could come back before incrementing requested job count. Increment must occur before Send(). - problem introduced when removing lock contention (lock would have blocked receiver previously) --- Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index 83b5ac561..cf5f0db68 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -668,9 +668,9 @@ void Server::FindNeedyClients() { continue; // Skip this worker for now } + cs->m_NumJobsRequested.Increment(); // Must be before Send() to ensure consistent counts msg.Send( cs->m_Connection ); } - cs->m_NumJobsRequested.Increment(); availableJobs--; anyJobsRequested = true; From 5b276019cc9b3e623870d99d718530eb406de544 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 21 May 2023 10:02:20 +0930 Subject: [PATCH 032/129] [Fix] Fix crash when .CompilerOutputPath is missing but required on ObjectList/Library --- Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index 814299cd1..e5649bcf5 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -233,6 +233,7 @@ ObjectListNode::ObjectListNode() if ( m_CompilerOutputPath.IsEmpty() ) { Error::Error_1101_MissingProperty( iter, function, AStackString<>( "CompilerOutputPath" ) ); + return false; } } From 87b4005f13cb58b285f90db789c296c1caa6b189 Mon Sep 17 00:00:00 2001 From: Harrison Ting Date: Sat, 27 May 2023 17:39:23 -0700 Subject: [PATCH 033/129] Fix docs to mark various function properties as optional that were incorreclty marked as required (#976) * Fix docs to mark various function properties as optional that were incorreclty marked as required * `DLL`/`Executable`: `Libraries2` * `Library`/`ObjectList`: `CompilerOutputPath` * `XCodeProject`: `XCodeOrganizationName` * Re-mark `CompilerOutputPath` as required, un-trim trailing whitespace --------- Co-authored-by: Harrison Ting --- Code/Tools/FBuild/Documentation/docs/functions/dll.html | 2 +- Code/Tools/FBuild/Documentation/docs/functions/executable.html | 2 +- .../Tools/FBuild/Documentation/docs/functions/xcodeproject.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/functions/dll.html b/Code/Tools/FBuild/Documentation/docs/functions/dll.html index 8627b39b5..2347d0cd6 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/dll.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/dll.html @@ -38,7 +38,7 @@

DLL

.LinkerOutput ; Output from linker .LinkerOptions ; Options to pass to linker .Libraries ; Libraries to link into DLL - .Libraries2 ; Secondary libraries to link into executable + .Libraries2 ; (optional) Secondary libraries to link into executable .LinkerLinkObjects ; (optional) Link objects used to make libs instead of libs (default true) .LinkerAssemblyResources ; (optional) List of assembly resources to use with %3 diff --git a/Code/Tools/FBuild/Documentation/docs/functions/executable.html b/Code/Tools/FBuild/Documentation/docs/functions/executable.html index c51579e3e..70fc83ef8 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/executable.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/executable.html @@ -38,7 +38,7 @@

Executable

.LinkerOutput ; Output from linker .LinkerOptions ; Options to pass to linker .Libraries ; Libraries to link into executable - .Libraries2 ; Secondary libraries to link into executable + .Libraries2 ; (optional) Secondary libraries to link into executable .LinkerLinkObjects ; (optional) Link objects used to make libs instead of libs (default false) .LinkerAssemblyResources ; (optional) List of assembly resources to use with %3 diff --git a/Code/Tools/FBuild/Documentation/docs/functions/xcodeproject.html b/Code/Tools/FBuild/Documentation/docs/functions/xcodeproject.html index e72e5a225..da84218c6 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/xcodeproject.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/xcodeproject.html @@ -297,7 +297,7 @@

XCodeProject

-

.XCodeOrganizationName - String - (Required)

+

.XCodeOrganizationName - String - (Optional)

The organization name which appears in the generated project can be set.

Example:
.XCodeOrganizationName = 'MyCompany'
From 8f378013f24c3e89073ad3e0eec5c6f04b114b65 Mon Sep 17 00:00:00 2001 From: Harrison Ting Date: Sat, 27 May 2023 18:10:05 -0700 Subject: [PATCH 034/129] Add the "FASTBuild Support" VS Code extension to the "3rd Party Downloads and Extensions" docs (#977) Co-authored-by: Harrison Ting --- Code/Tools/FBuild/Documentation/docs/download.html | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/Tools/FBuild/Documentation/docs/download.html b/Code/Tools/FBuild/Documentation/docs/download.html index 6a1afa7d7..46aaaed01 100644 --- a/Code/Tools/FBuild/Documentation/docs/download.html +++ b/Code/Tools/FBuild/Documentation/docs/download.html @@ -75,6 +75,7 @@

Editor Integration

  • vim-fastbuild - Syntax highlighting for vim by dummyunit
  • FASTBuild-Sublime - Syntax highlighting for Sublime Text 3 by Manuzor
    • +
  • FASTBuild Support - Syntax highlighting, go-to definition, find references, hover to see variable values, show syntax errors, and more for Visual Studio Code by Harrison Ting
From 325374cba0e27d3f7b606989bf072373edddf39a Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 28 May 2023 11:02:34 +0930 Subject: [PATCH 035/129] Update styling for 3rd Party "Editor Integration" links --- Code/Tools/FBuild/Documentation/docs/download.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/download.html b/Code/Tools/FBuild/Documentation/docs/download.html index 46aaaed01..617004896 100644 --- a/Code/Tools/FBuild/Documentation/docs/download.html +++ b/Code/Tools/FBuild/Documentation/docs/download.html @@ -73,9 +73,9 @@

Unreal Engine

Editor Integration

    -
  • vim-fastbuild - Syntax highlighting for vim by dummyunit
    • -
  • FASTBuild-Sublime - Syntax highlighting for Sublime Text 3 by Manuzor
    • -
  • FASTBuild Support - Syntax highlighting, go-to definition, find references, hover to see variable values, show syntax errors, and more for Visual Studio Code by Harrison Ting
    • +
  • Sublime - Syntax highlighting for Sublime Text 3 by Manuzor
    • +
  • vim - Syntax highlighting for vim by dummyunit
    • +
  • VSCode - Syntax highlighting, go-to definition, find references, hover to see variable values, and much more by Harrison Ting
From 907d3a6b47ec05405cb7c608569f9bcfb2824ce6 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 28 May 2023 12:42:48 +0930 Subject: [PATCH 036/129] Worker periodic restart is disabled by default and can be activated via -periodicrestart --- .../Tools/FBuild/Documentation/docs/options.html | 11 +++++++++++ .../FBuild/FBuildWorker/FBuildWorkerOptions.cpp | 10 +++++++++- .../FBuild/FBuildWorker/FBuildWorkerOptions.h | 3 +++ Code/Tools/FBuild/FBuildWorker/Main.cpp | 2 +- Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp | 16 ++++++++++------ Code/Tools/FBuild/FBuildWorker/Worker/Worker.h | 3 ++- 6 files changed, 36 insertions(+), 9 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/options.html b/Code/Tools/FBuild/Documentation/docs/options.html index f84ca135f..3440cda31 100644 --- a/Code/Tools/FBuild/Documentation/docs/options.html +++ b/Code/Tools/FBuild/Documentation/docs/options.html @@ -243,6 +243,10 @@

FBuildWorker.exe

-nosubprocess Don't spawn as a sub-process. + + -periodicrestart + Restart worker every 4 hours. + @@ -718,6 +722,13 @@

FBuildWorker.exe Detailed

The "-nosubprocess" option suppresses this behaviour.

+
-periodicrestart
+
+

Restart worker every 4 hours.

+

If worker reliability issues are encountered, perhaps due to uncontrolable factors such as OS instability, network driver issues or as yet unresolved FASTBuild bugs, the worker can be instructed to periodically restart itself as a potential workaround.

+
+ +
© 2012-2023 Franta Fulin
diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp index 82f400ee5..3eeb3ccba 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp @@ -31,7 +31,8 @@ FBuildWorkerOptions::FBuildWorkerOptions() : m_OverrideWorkMode( false ), m_WorkMode( WorkerSettings::WHEN_IDLE ), m_MinimumFreeMemoryMiB( 0 ), - m_ConsoleMode( false ) + m_ConsoleMode( false ), + m_PeriodicRestart( false ) { #ifdef __LINUX__ m_ConsoleMode = true; // Only console mode supported on Linux @@ -114,6 +115,11 @@ bool FBuildWorkerOptions::ProcessCommandLine( const AString & commandLine ) m_OverrideWorkMode = true; continue; } + else if ( token == "-periodicrestart" ) + { + m_PeriodicRestart = true; + continue; + } #if defined( __WINDOWS__ ) else if ( token.BeginsWith( "-minfreememory=" ) ) { @@ -175,6 +181,8 @@ void FBuildWorkerOptions::ShowUsageError() " Set minimum free memory (MiB) required to accept work.\n" " -nosubprocess\n" " (Windows) Don't spawn a sub-process worker copy.\n" + " -periodicrestart\n" + " Worker will restart every 4 hours.\n" "---------------------------------------------------------------------------\n" ; diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.h b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.h index c01ffe0b6..7f016555e 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.h +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.h @@ -39,6 +39,9 @@ class FBuildWorkerOptions // Console mode bool m_ConsoleMode; + // Other + bool m_PeriodicRestart; + private: void ShowUsageError(); }; diff --git a/Code/Tools/FBuild/FBuildWorker/Main.cpp b/Code/Tools/FBuild/FBuildWorker/Main.cpp index c4adb7cb3..262ce780b 100644 --- a/Code/Tools/FBuild/FBuildWorker/Main.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Main.cpp @@ -126,7 +126,7 @@ int Main( const AString & args ) // start the worker and wait for it to be closed int ret; { - Worker worker( args, options.m_ConsoleMode ); + Worker worker( args, options.m_ConsoleMode, options.m_PeriodicRestart ); if ( options.m_OverrideCPUAllocation ) { WorkerSettings::Get().SetNumCPUsToUse( options.m_CPUAllocation ); diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp index f717cf1b5..3aaf8cfc2 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp @@ -41,8 +41,9 @@ // CONSTRUCTOR //------------------------------------------------------------------------------ -Worker::Worker( const AString & args, bool consoleMode ) +Worker::Worker( const AString & args, bool consoleMode, bool periodicRestart ) : m_ConsoleMode( consoleMode ) + , m_PeriodicRestart( periodicRestart ) , m_MainWindow( nullptr ) , m_ConnectionPool( nullptr ) , m_NetworkStartupHelper( nullptr ) @@ -456,12 +457,15 @@ void Worker::CheckIfRestartNeeded() } // Check if periodic restart time has been reached - const float periodicRestartSecs = ( 4.0f * 60.0f * 60.0f ); // 4 hours - if ( m_PeriodicRestartTimer.GetElapsed() > periodicRestartSecs ) + if ( m_PeriodicRestart ) { - m_RestartNeeded = true; - JobQueueRemote::Get().SignalStopWorkers(); - return; + const float periodicRestartSecs = ( 4.0f * 60.0f * 60.0f ); // 4 hours + if ( m_PeriodicRestartTimer.GetElapsed() > periodicRestartSecs ) + { + m_RestartNeeded = true; + JobQueueRemote::Get().SignalStopWorkers(); + return; + } } // get the current last write time diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h index 731cb2888..afa86e2ba 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h @@ -27,7 +27,7 @@ class WorkerSettings; class Worker : public Singleton { public: - explicit Worker( const AString & args, bool consoleMode ); + explicit Worker( const AString & args, bool consoleMode, bool periodicRestart ); ~Worker(); int32_t Work(); @@ -50,6 +50,7 @@ class Worker : public Singleton void ErrorMessage( MSVC_SAL_PRINTF const char * fmtString, ... ) const FORMAT_STRING( 2, 3 ); bool m_ConsoleMode; + bool m_PeriodicRestart; WorkerWindow * m_MainWindow; Server * m_ConnectionPool; NetworkStartupHelper * m_NetworkStartupHelper; From 021dc6137aea6eeae9cba39dd5fee4080016133b Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 28 May 2023 14:56:49 +0930 Subject: [PATCH 037/129] v1.10 --- .../FBuild/Documentation/docs/changelog.html | 39 ++++++++++++++++ .../FBuild/Documentation/docs/download.html | 44 ++++++++++++------- .../Tools/FBuild/Documentation/docs/home.html | 6 ++- Code/Tools/FBuild/FBuildCore/FBuildVersion.h | 4 +- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/changelog.html b/Code/Tools/FBuild/Documentation/docs/changelog.html index e8aed1bd8..2486d3c9f 100644 --- a/Code/Tools/FBuild/Documentation/docs/changelog.html +++ b/Code/Tools/FBuild/Documentation/docs/changelog.html @@ -26,6 +26,45 @@

Changelog

+
+ v1.10 (28th-May-2023) +
+
+ Notes +
    +
  • DB Version Changed : clean build will occur
    • +
  • Cache version changed : cache will be re-populated
    • +
+ New +
    +
  • [OSX] Apple Silicon is supported natively using Universal binaries
    • +
+ Fixes +
    +
  • Distributed Compilation
    • +
      +
    • Prevent Worker over-requesting jobs when there are multiple Clients
      • +
    +
  • [Linux][OSX] Fix file/folder permissions being exessively permissive (Thanks to Pete Lewis)
    • +
  • Chained ObjectLists correctly handle duplicate names in sub-directories (Thanks to Dandielo)
    • +
  • Fix crash when .CompilerOutputPath is missing but required on ObjectList/Library
    • +
+ Improvements +
    +
  • Distributed Compilation
    • +
      +
    • Toolchain management performance improved slightly when toolchain changes
      • +
    • Remove some unecessary lock contention in FBuildWorker
      • +
    • FBuildWorker can be set to restart every 4 hours (-periodicrestart)
      • +
    • Eliminate various instances of lock contention in FBuildWorker
      • +
    +
  • Cache performance improved slightly for check, store and retrieve operations
    • +
  • Performance of various build steps improved slightly
    • +
  • Database validation improved, reducing startup/shutdown time slightly
    • +
  • Fix documenation errors for some optional fields (Thanks to Harrison Ting)
    • +
+
+
v1.09 (5th-Mar-2023)
diff --git a/Code/Tools/FBuild/Documentation/docs/download.html b/Code/Tools/FBuild/Documentation/docs/download.html index 617004896..ad96b027d 100644 --- a/Code/Tools/FBuild/Documentation/docs/download.html +++ b/Code/Tools/FBuild/Documentation/docs/download.html @@ -31,28 +31,30 @@

Download

- Current Version - v1.09 + Current Version

See the changelog for a detailed list of changes.

-Binaries - -Source - +
+ + + + + + + + +
VersionWindowsOS XLinuxSource
v1.10x64x64 & ARMx64Zip | + GitHub
+
+ Syntax Highlighting
@@ -85,6 +87,16 @@

Editor Integration

+ + + + + + + + + diff --git a/Code/Tools/FBuild/Documentation/docs/home.html b/Code/Tools/FBuild/Documentation/docs/home.html index 98efeaa09..e766d5477 100644 --- a/Code/Tools/FBuild/Documentation/docs/home.html +++ b/Code/Tools/FBuild/Documentation/docs/home.html @@ -38,12 +38,14 @@
- FASTBuild v1.09 released!  (5-March-2023) + FASTBuild v1.10 released!  (28-May-2023)
An updated version of FASTBuild can now be downloaded, featuring:
    -
  • Several bug fixes, reliability improvements and performance enhancements
    • +
  • Native support for Apple Silicon (ARM) Macs, significantly improving performance
    • +
  • Various performance enhancements
    • +
  • Some misc bug fixes and reliability improvements
  • Details...
diff --git a/Code/Tools/FBuild/FBuildCore/FBuildVersion.h b/Code/Tools/FBuild/FBuildCore/FBuildVersion.h index aa48f2e1f..d227ef6a4 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildVersion.h +++ b/Code/Tools/FBuild/FBuildCore/FBuildVersion.h @@ -4,8 +4,8 @@ // Defines //------------------------------------------------------------------------------ -#define FBUILD_VERSION_STRING "v1.09" -#define FBUILD_VERSION (uint32_t)109 +#define FBUILD_VERSION_STRING "v1.10" +#define FBUILD_VERSION (uint32_t)110 #if defined( __WINDOWS__ ) #define FBUILD_VERSION_PLATFORM "Windows" #elif defined( __APPLE__ ) From 0db73c1899d6b9620db2e3d2853faf928fed79ac Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 28 May 2023 15:05:06 +0930 Subject: [PATCH 038/129] Integrate 'dev' to 'main' for v1.10 release commit 021dc6137aea6eeae9cba39dd5fee4080016133b Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun May 28 14:56:49 2023 +0930 v1.10 commit 907d3a6b47ec05405cb7c608569f9bcfb2824ce6 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun May 28 12:42:48 2023 +0930 Worker periodic restart is disabled by default and can be activated via -periodicrestart commit 325374cba0e27d3f7b606989bf072373edddf39a Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun May 28 11:02:34 2023 +0930 Update styling for 3rd Party "Editor Integration" links commit 8f378013f24c3e89073ad3e0eec5c6f04b114b65 Author: Harrison Ting Date: Sat May 27 18:10:05 2023 -0700 Add the "FASTBuild Support" VS Code extension to the "3rd Party Downloads and Extensions" docs (#977) Co-authored-by: Harrison Ting commit 87b4005f13cb58b285f90db789c296c1caa6b189 Author: Harrison Ting Date: Sat May 27 17:39:23 2023 -0700 Fix docs to mark various function properties as optional that were incorreclty marked as required (#976) * Fix docs to mark various function properties as optional that were incorreclty marked as required * `DLL`/`Executable`: `Libraries2` * `Library`/`ObjectList`: `CompilerOutputPath` * `XCodeProject`: `XCodeOrganizationName` * Re-mark `CompilerOutputPath` as required, un-trim trailing whitespace --------- Co-authored-by: Harrison Ting commit 5b276019cc9b3e623870d99d718530eb406de544 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun May 21 10:02:20 2023 +0930 [Fix] Fix crash when .CompilerOutputPath is missing but required on ObjectList/Library commit c78009ad8b2a3c11427735dbe7cf912521f9675f Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 17:37:17 2023 +0930 Fix requested job count becoming negative under heavy load - result of request could come back before incrementing requested job count. Increment must occur before Send(). - problem introduced when removing lock contention (lock would have blocked receiver previously) commit d055722eb0c99f46973e55d0c9688f0cc615dd0f Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 16:31:13 2023 +0930 [Improvement] Eliminate various instances of lock contention in FBuildWorker commit 7557f790b19c50233d31f58a63cae6152c798db3 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 15:02:48 2023 +0930 Mutex: Add TryMutexHolder to support RAII patterns with speculatively acquired Mutexes commit 51ff53bed76f300779cdbb9e6bdbb101b50ab0a8 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 14:48:58 2023 +0930 [Improvment] Remove some unecessary lock contention in FBuildWorker - avoid locking the worker list unnecessarily when not accepting work commit 0348efbc287e228b4fa690755b679a3000bd9f67 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 14:30:18 2023 +0930 [Improvement] FBuildWorker will restart every 4 hours to improve reliability - this helps ensure intermittent issues don't impact overall worker pool reliability commit 2f7bf468581efa65c63428f15651d5dbf23d3d9e Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 12:42:07 2023 +0930 Post-intergation: Chained ObjectList fix commit 5e15f13253a69e7a7483d030a4340cb2ad3c6d05 Author: Dandielo Date: Sun Apr 30 11:30:03 2023 +0900 Fix collision errors on chained ObjectLists (#966) * Chained ObjectLists now properly handle sub-directories. * Fix test by increasing number of objects seend and built. commit e6487e754df4878d91b6c578b965a4371054e538 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 11:24:31 2023 +0930 git CI uses Universal binary targets for OSX commit 7a5221053a21b149a3e70eaf3e0241c5d475245e Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 11:06:37 2023 +0930 [Improvement][OSX] Add native Apple Silicon support using Universal binaries - combine existing arm and x64 executables together into universal binaries via new "OSX" targets - gated on availability of Clang12 or newer commit 5fdc9fa6c149a14c5337f43e06dbaec1ad1828af Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 30 10:57:41 2023 +0930 OSX: Simplify Clang12 configs - de-duplicate config common to x64 and ARM commit f9504642d8dc015a3f56d0713adb22dba2f053ec Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Apr 29 12:39:21 2023 +0930 OSX: Use Clang12 to build FASTBuild by default commit 326aa7d1fa5ea90425de7131d961d8cf7320c104 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Apr 29 12:22:40 2023 +0930 Tidy up architecture defines - use compile built-ins for detection of X64 vs ARM instead of setting those in the bff files - this is necessary to allow multiple architectures to be generated in one compiler invocation for OSX Universal Binaries commit b4de4c42b44b57342fb582585d0d7d9cbdfdcaab Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Apr 29 11:47:35 2023 +0930 Remove remnants of 32bits support for compiling FASTBuild - FASTBuild has been an exclusively 64bit process for sometime - The processes it can spawn are unaffected commit 755d5be656ffe36cec30b4ed99b933eff7c4f858 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Apr 15 16:42:09 2023 +0930 [Fix] Prevent Worker over-requesting jobs when there are multiple Clients commit c9794e17483ccf49e7307c3e0eb5551273bbbc8c Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Apr 15 15:07:20 2023 +0930 Mutex: Add TryLock functionality - speculative lock acquisition that returns immediately on failure commit 9e7808c7a598ff29f663768798d5865f1b6d4cea Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Apr 15 14:38:40 2023 +0930 Mutex: Small code cleanup (no functional changes) commit 873b1ccf202259a407a0dbfbc604b329d86ef5d5 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 9 12:07:46 2023 +0930 Remove Linux GCC 7 from github actions CI - seems like either GCC7 or the version of ubuntu used causes the github action to be stuck in queue forever commit 077bf93a163663da89ecc8475b57b30707cf5539 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Apr 9 11:24:53 2023 +0930 Post-integration cleanup for Linux/OSX FileIO permissions fixes - minor style fixes commit a033b1a757068174afb5116acff58e2a38da6bfa Author: Pete Lewis Date: Sat Apr 8 18:35:42 2023 -0700 Fix Linux and MacOS file permissions: (#956) * SetExecutable: preserve existing RW permissions; just add X * SetReadOnly: when removing the flag, remove W entirely; when adding the flag, only set for USR * mkdir: Create directories with 0755 permissions commit 7119e67d9759e9c4276850e7e419d98425531123 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Mar 26 13:22:21 2023 +1030 BFFVariables: Variables track their origin to facilitate improved errors in the future - We keep a pointer to the BFFToken responsible for the creation of the variable commit 24742946ba083d3f239b71458984b5d9077a4238 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Mar 19 11:50:10 2023 +1030 Various small performance improvements through use of xxHash3 [Improvement] Cache performance improved slightly for check, store and retrieve operations [Improvement] Toolchain management performance improved slightly when toolchain changes [Improvement] Performance of various build steps improved slightly [Improvement] Database validation improved slightly, reducing startup/shutdown time slightly - switch 64bit hashes to xxHash3 which is ~2x-3x faster than xxHash [Note] DB version changed [Note] Cache version changed commit 22b92f651ac98a287d995213f9b00c206ffd2bd0 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Mar 19 11:48:06 2023 +1030 BFFFuzzer: Fix linking with xxHash3 commit 1e881ed1ea45691ff414d47abe23f5aae87e7ae2 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Mar 19 10:51:44 2023 +1030 Core: Remove most of deprecated Thread API - use new API where possible - trim legacy API to only what remains in use - make Thread internally directly use what it needs instead of forwarding to the deprecated API commit 678cadb89582234683d2fc41e617e53047d3cc37 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Mar 18 16:37:16 2023 +1030 Threads: Replace most deprecated Thread API usage with newer/safer API - Also activate Thread test that was not enabled commit d8e6d1e9c888531e525983fbd95134983014fec8 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Mar 12 15:06:36 2023 +1030 xxHash3: Expose xxHash3 64 for future use (2-3x faster than regular xxhash 64) commit f6f3ea27aae09e7d2d347e6ecf0bfd96c8a8dd46 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun Mar 12 15:05:53 2023 +1030 Add xxHash v0.8.1 (not yet used) commit f3fe70e819b4da35d3aae71755d6ca0ce0bca8e4 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Mar 11 16:10:46 2023 +1030 Remove VS2022 from CI coverage - some configuration issues need to be resolved commit f675ad9100cff8b970b5d9632cdc7fb6bd22afc2 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Mar 11 16:07:03 2023 +1030 Add VS2022 to CI coverage commit 4c876e6d59576509756b9b895df28ccfd98135d1 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Mar 11 15:59:23 2023 +1030 Switch CI "checkout" action to v3 - update to avoid deprecation warnings about the old action using nodejs 12 commit 3d4ec0e420eae1ccf8d0c0e1b5f29c54b1d61f98 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Mar 11 15:48:14 2023 +1030 Update OSX CI to use macos11 - github reports that 10.15 is deprecated commit 387e713f8ff69c79d6949d84b86f23b6280aa800 Author: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat Mar 11 15:45:49 2023 +1030 Update dev CI to use v1.09 --- .github/actions/DownloadFBuild/action.yml | 2 +- .github/workflows/Linux.yml | 6 +- .github/workflows/OSX.yml | 8 +- .github/workflows/Windows.yml | 2 +- Code/.FASTBuild/HelperFunctions.bff | 37 +- Code/Core/CoreTest/CoreTest.bff | 1 + Code/Core/CoreTest/TestMain.cpp | 1 + Code/Core/CoreTest/Tests/TestAtomic.cpp | 11 +- Code/Core/CoreTest/Tests/TestHash.cpp | 25 + Code/Core/CoreTest/Tests/TestMutex.cpp | 101 +- Code/Core/CoreTest/Tests/TestSemaphore.cpp | 8 +- .../Tests/TestSmallBlockAllocator.cpp | 18 +- .../CoreTest/Tests/TestTCPConnectionPool.cpp | 11 +- Code/Core/CoreTest/Tests/TestThread.cpp | 3 + Code/Core/CoreTest/Tests/TestTimer.cpp | 2 +- Code/Core/Env/Types.h | 17 +- Code/Core/FileIO/FileIO.cpp | 26 +- Code/Core/Math/xxHash.h | 21 + Code/Core/Network/Network.cpp | 15 +- Code/Core/Network/TCPConnectionPool.cpp | 20 +- Code/Core/Process/Mutex.cpp | 29 +- Code/Core/Process/Mutex.h | 41 +- Code/Core/Process/Thread.cpp | 248 +- Code/Core/Process/Thread.h | 14 +- Code/Core/Time/Timer.cpp | 4 +- Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff | 1 + .../FBuild/Documentation/docs/changelog.html | 39 + .../FBuild/Documentation/docs/download.html | 49 +- .../Documentation/docs/functions/dll.html | 2 +- .../docs/functions/executable.html | 2 +- .../docs/functions/xcodeproject.html | 2 +- .../Tools/FBuild/Documentation/docs/home.html | 6 +- .../FBuild/Documentation/docs/options.html | 11 + Code/Tools/FBuild/FBuild/FBuild.bff | 14 + Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp | 4 +- .../Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp | 4 +- .../Tools/FBuild/FBuildCore/BFF/BFFParser.cpp | 80 +- Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h | 6 +- .../FBuild/FBuildCore/BFF/BFFStackFrame.cpp | 57 +- .../FBuild/FBuildCore/BFF/BFFStackFrame.h | 17 +- .../FBuild/FBuildCore/BFF/BFFVariable.cpp | 51 +- .../Tools/FBuild/FBuildCore/BFF/BFFVariable.h | 19 +- .../BFF/Functions/FunctionForEach.cpp | 6 +- .../BFF/Functions/FunctionUsing.cpp | 4 +- .../FBuildCore/BFF/Tokenizer/BFFToken.cpp | 9 + .../FBuildCore/BFF/Tokenizer/BFFToken.h | 7 + Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp | 2 +- .../FBuild/FBuildCore/Cache/LightCache.cpp | 10 +- Code/Tools/FBuild/FBuildCore/FBuildVersion.h | 4 +- .../FBuild/FBuildCore/Graph/CopyDirNode.cpp | 2 +- .../FBuildCore/Graph/DirectoryListNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/NodeGraph.cpp | 6 +- .../Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 2 +- .../FBuildCore/Graph/ObjectListNode.cpp | 5 +- .../FBuild/FBuildCore/Graph/ObjectListNode.h | 1 + .../FBuild/FBuildCore/Graph/ObjectNode.cpp | 6 +- .../Tools/FBuild/FBuildCore/Graph/SLNNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/UnityNode.cpp | 4 +- .../FBuildCore/Graph/VCXProjectNode.cpp | 2 +- .../FBuildCore/Graph/XCodeProjectNode.cpp | 6 +- .../FBuildCore/Helpers/BuildProfiler.cpp | 12 +- .../FBuild/FBuildCore/Helpers/BuildProfiler.h | 2 +- .../FBuildCore/Helpers/ToolManifest.cpp | 2 +- .../FBuild/FBuildCore/Protocol/Client.cpp | 10 +- .../Tools/FBuild/FBuildCore/Protocol/Client.h | 2 +- .../FBuild/FBuildCore/Protocol/Server.cpp | 84 +- .../Tools/FBuild/FBuildCore/Protocol/Server.h | 10 +- .../FBuildCore/WorkerPool/WorkerThread.cpp | 9 +- .../FBuildCore/WorkerPool/WorkerThread.h | 2 + .../ObjectListChaining/A/file.1.cpp | 0 Code/Tools/FBuild/FBuildTest/FBuildTest.bff | 1 + .../FBuildTest/Tests/TestDistributed.cpp | 6 +- .../FBuildTest/Tests/TestFastCancel.cpp | 6 +- .../FBuild/FBuildTest/Tests/TestGraph.cpp | 2 +- .../FBuildTest/Tests/TestNodeReflection.cpp | 60 +- .../FBuildTest/Tests/TestObjectList.cpp | 6 +- .../FBuildTest/Tests/TestVariableStack.cpp | 13 +- .../FBuild/FBuildWorker/FBuildWorker.bff | 16 +- .../FBuildWorker/FBuildWorkerOptions.cpp | 10 +- .../FBuild/FBuildWorker/FBuildWorkerOptions.h | 3 + Code/Tools/FBuild/FBuildWorker/Main.cpp | 2 +- .../FBuild/FBuildWorker/Worker/Worker.cpp | 31 +- .../Tools/FBuild/FBuildWorker/Worker/Worker.h | 8 +- Code/fbuild.bff | 50 +- External/SDK/Clang/Clang.bff | 4 +- External/SDK/Clang/Linux/Clang10.bff | 1 - External/SDK/Clang/Linux/Clang3.bff | 1 - External/SDK/Clang/Linux/Clang6.bff | 1 - External/SDK/Clang/Linux/Clang_CI.bff | 1 - External/SDK/Clang/OSX/Clang12.bff | 67 +- External/SDK/Clang/OSX/Clang8.bff | 1 - External/SDK/Clang/OSX/Clang_CI.bff | 66 +- External/SDK/GCC/Linux/GCC4.bff | 1 - External/SDK/GCC/Linux/GCC7.bff | 1 - External/SDK/GCC/Linux/GCC9.bff | 1 - External/SDK/GCC/Linux/GCC_CI.bff | 1 - External/SDK/Windows/Windows10SDK.bff | 2 - External/xxHash/0.8.1/CHANGELOG | 71 + External/xxHash/0.8.1/LICENSE | 26 + External/xxHash/0.8.1/README.md | 236 + External/xxHash/0.8.1/libxxhash.pc.in | 15 + External/xxHash/0.8.1/xxh3.h | 55 + External/xxHash/0.8.1/xxh_x86dispatch.c | 770 +++ External/xxHash/0.8.1/xxh_x86dispatch.h | 86 + External/xxHash/0.8.1/xxhash.c | 43 + External/xxHash/0.8.1/xxhash.h | 5580 +++++++++++++++++ External/xxHash/xxHash.bff | 119 + 107 files changed, 7939 insertions(+), 672 deletions(-) create mode 100644 Code/Tools/FBuild/FBuildTest/Data/TestObjectList/ObjectListChaining/A/file.1.cpp create mode 100644 External/xxHash/0.8.1/CHANGELOG create mode 100644 External/xxHash/0.8.1/LICENSE create mode 100644 External/xxHash/0.8.1/README.md create mode 100644 External/xxHash/0.8.1/libxxhash.pc.in create mode 100644 External/xxHash/0.8.1/xxh3.h create mode 100644 External/xxHash/0.8.1/xxh_x86dispatch.c create mode 100644 External/xxHash/0.8.1/xxh_x86dispatch.h create mode 100644 External/xxHash/0.8.1/xxhash.c create mode 100644 External/xxHash/0.8.1/xxhash.h create mode 100644 External/xxHash/xxHash.bff diff --git a/.github/actions/DownloadFBuild/action.yml b/.github/actions/DownloadFBuild/action.yml index f2367f941..dbd0d17be 100644 --- a/.github/actions/DownloadFBuild/action.yml +++ b/.github/actions/DownloadFBuild/action.yml @@ -5,7 +5,7 @@ inputs: version: description: Version to download required: false - default: 1.08 + default: 1.09 runs: using: composite diff --git a/.github/workflows/Linux.yml b/.github/workflows/Linux.yml index 27cd0109c..88639f3c9 100644 --- a/.github/workflows/Linux.yml +++ b/.github/workflows/Linux.yml @@ -17,10 +17,6 @@ jobs: gcc: 10 clang: 12 os: ubuntu-20.04 - - cfg: Build - gcc: 7 - clang: 9 - os: ubuntu-18.04 - cfg: ASan gcc: 10 clang: 12 @@ -41,7 +37,7 @@ jobs: continue-on-error: ${{ matrix.can-fail || false }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/DownloadFBuild diff --git a/.github/workflows/OSX.yml b/.github/workflows/OSX.yml index 3e1f7618d..854321d42 100644 --- a/.github/workflows/OSX.yml +++ b/.github/workflows/OSX.yml @@ -14,13 +14,13 @@ jobs: matrix: include: - cfg: Build & Tests - os: macos-10.15 + os: macos-11 name: OSX (${{ matrix.cfg }}) runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/DownloadFBuild @@ -43,7 +43,7 @@ jobs: - name: Build if: ${{ startsWith(matrix.cfg, 'Build') }} - run: ${FBUILD_PATH} -nostoponerror -summary All-x64OSX-{Debug,Profile,Release} + run: ${FBUILD_PATH} -nostoponerror -summary All-OSX-{Debug,Profile,Release} - name: Tests # -j1 on CI nodes avoids timeouts (CI nodes have only 2 cores) @@ -52,4 +52,4 @@ jobs: - name: Build (NoUnity) if: ${{ startsWith(matrix.cfg, 'Build') }} - run: ${FBUILD_PATH} -nostoponerror -summary -nounity -clean All-x64OSX-Debug + run: ${FBUILD_PATH} -nostoponerror -summary -nounity -clean All-OSX-Debug diff --git a/.github/workflows/Windows.yml b/.github/workflows/Windows.yml index 22b80e9da..41f52d23a 100644 --- a/.github/workflows/Windows.yml +++ b/.github/workflows/Windows.yml @@ -27,7 +27,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: ./.github/actions/DownloadFBuild diff --git a/Code/.FASTBuild/HelperFunctions.bff b/Code/.FASTBuild/HelperFunctions.bff index f74179a2e..62102fd1d 100644 --- a/Code/.FASTBuild/HelperFunctions.bff +++ b/Code/.FASTBuild/HelperFunctions.bff @@ -59,9 +59,16 @@ function CreateCommonAliases( .ProjectName ) Alias( '$ProjectName$-TSan' ) { .Targets = { '$ProjectName$-X64Linux-TSan', '$ProjectName$-X64ClangLinux-TSan' } } #endif #if __OSX__ - Alias( '$ProjectName$-Debug' ) { .Targets = { '$ProjectName$-X64OSX-Debug' } } - Alias( '$ProjectName$-Profile' ) { .Targets = { '$ProjectName$-X64OSX-Profile' } } - Alias( '$ProjectName$-Release' ) { .Targets = { '$ProjectName$-X64OSX-Release' } } + #if CLANG_SUPPORTS_ARMOSX + // Universal Binaries + Alias( '$ProjectName$-Debug' ) { .Targets = { '$ProjectName$-OSX-Debug' } } + Alias( '$ProjectName$-Profile' ) { .Targets = { '$ProjectName$-OSX-Profile' } } + Alias( '$ProjectName$-Release' ) { .Targets = { '$ProjectName$-OSX-Release' } } + #else + Alias( '$ProjectName$-Debug' ) { .Targets = { '$ProjectName$-X64OSX-Debug' } } + Alias( '$ProjectName$-Profile' ) { .Targets = { '$ProjectName$-X64OSX-Profile' } } + Alias( '$ProjectName$-Release' ) { .Targets = { '$ProjectName$-X64OSX-Release' } } + #endif #endif // Per-Platform @@ -70,6 +77,10 @@ function CreateCommonAliases( .ProjectName ) Alias( '$ProjectName$-X64Linux' ) { .Targets = { '$ProjectName$-X64Linux-Debug', '$ProjectName$-X64Linux-Release', '$ProjectName$-X64Linux-Profile', '$ProjectName$-X64Linux-ASan', '$ProjectName$-X64Linux-TSan' } } Alias( '$ProjectName$-X64ClangLinux' ) { .Targets = { '$ProjectName$-X64ClangLinux-Debug', '$ProjectName$-X64ClangLinux-Release', '$ProjectName$-X64ClangLinux-Profile', '$ProjectName$-X64ClangLinux-ASan', '$ProjectName$-X64ClangLinux-MSan', '$ProjectName$-X64ClangLinux-TSan' } } Alias( '$ProjectName$-X64OSX' ) { .Targets = { '$ProjectName$-X64OSX-Debug', '$ProjectName$-X64OSX-Release', '$ProjectName$-X64OSX-Profile' } } + #if CLANG_SUPPORTS_ARMOSX + Alias( '$ProjectName$-ARMOSX' ) { .Targets = { '$ProjectName$-ARMOSX-Debug', '$ProjectName$-ARMOSX-Release', '$ProjectName$-ARMOSX-Profile' } } + Alias( '$ProjectName$-OSX' ) { .Targets = { '$ProjectName$-OSX-Debug', '$ProjectName$-OSX-Release', '$ProjectName$-OSX-Profile' } } + #endif // All Alias( '$ProjectName$' ) @@ -159,4 +170,24 @@ function CreateVCXProject_Exe( .Name CreateVCXProject_Lib( .Name, .Path, .Configs, .ExtraOptions ) } + +// CreateUniversalBinary +//------------------------------------------------------------------------------ +#if CLANG_SUPPORTS_ARMOSX + function CreateUniversalBinary( .ProjectName + .ProjectPath + .BuildConfigName + .OutputBase ) + { + Exec( '$ProjectName$-Exe-OSX-$BuildConfigName$' ) + { + .ExecExecutable = '/usr/bin/lipo' + .ExecInput = { '$ProjectName$-Exe-x64OSX-$BuildConfigName$', '$ProjectName$-Exe-ARMOSX-$BuildConfigName$' } + .ExecOutput = '$OutputBase$/OSX-$BuildConfigName$/$ProjectPath$/$ProjectName$' + .ExecArguments = '-create -output %2 %1' + } + Alias( '$ProjectName$-OSX-$BuildConfigName$' ) { .Targets = '$ProjectName$-Exe-OSX-$BuildConfigName$' } + } +#endif + //------------------------------------------------------------------------------ diff --git a/Code/Core/CoreTest/CoreTest.bff b/Code/Core/CoreTest/CoreTest.bff index ed49de3f2..1a177e7c2 100644 --- a/Code/Core/CoreTest/CoreTest.bff +++ b/Code/Core/CoreTest/CoreTest.bff @@ -52,6 +52,7 @@ 'TestFrameWork-Lib-$Platform$-$BuildConfigName$' 'Core-Lib-$Platform$-$BuildConfigName$' 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/$ProjectName$$ExeExtension$' #if __WINDOWS__ diff --git a/Code/Core/CoreTest/TestMain.cpp b/Code/Core/CoreTest/TestMain.cpp index e1b600f21..670a32d01 100644 --- a/Code/Core/CoreTest/TestMain.cpp +++ b/Code/Core/CoreTest/TestMain.cpp @@ -28,6 +28,7 @@ int main( int, char *[] ) REGISTER_TESTGROUP( TestSmallBlockAllocator ) REGISTER_TESTGROUP( TestSystemMutex ) REGISTER_TESTGROUP( TestTestTCPConnectionPool ) + REGISTER_TESTGROUP( TestThread ) REGISTER_TESTGROUP( TestTimer ) REGISTER_TESTGROUP( TestUnorderedMap ) diff --git a/Code/Core/CoreTest/Tests/TestAtomic.cpp b/Code/Core/CoreTest/Tests/TestAtomic.cpp index 666ea92f8..f88eb643f 100644 --- a/Code/Core/CoreTest/Tests/TestAtomic.cpp +++ b/Code/Core/CoreTest/Tests/TestAtomic.cpp @@ -51,19 +51,14 @@ class AtomicTestHelper TEST_ASSERT( m_Count2.Load() == initialValue ); // Spawn thread - Thread::ThreadHandle h = Thread::CreateThread( ThreadWrapper, - "AtomicTestHelper", - ( 64 * KILOBYTE ), - this ); + Thread t; + t.Start( ThreadWrapper, "AtomicTestHelper", this ); // Do works locally that mirrors the thread DoWork(); // Join thread - bool timedOut = false; - Thread::WaitForThread( h, 1000, timedOut ); - TEST_ASSERT( timedOut == false ); - Thread::CloseHandle( h ); + t.Join(); // Check expected results TEST_ASSERT( AtomicLoadRelaxed( &m_Count ) == expectedResult ); diff --git a/Code/Core/CoreTest/Tests/TestHash.cpp b/Code/Core/CoreTest/Tests/TestHash.cpp index 3041ae1f7..7d42c3c7b 100644 --- a/Code/Core/CoreTest/Tests/TestHash.cpp +++ b/Code/Core/CoreTest/Tests/TestHash.cpp @@ -104,6 +104,15 @@ void TestHash::CompareHashTimes_Large() const OUTPUT( "xxHash-64 : %2.3fs @ %6.3f GiB/s (hash: %016" PRIx64 ")\n", (double)time, (double)speed, crc ); } + // xxHash3_64 + { + const Timer t; + const uint64_t crc = xxHash3::Calc64( data.Get(), dataSize ); + const float time = t.GetElapsed(); + const float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; + OUTPUT( "xxHash3-64 : %2.3fs @ %6.3f GiB/s (hash: %016" PRIx64 ")\n", (double)time, (double)speed, crc ); + } + // CRC32 - 8x8 slicing { const Timer t; @@ -192,6 +201,22 @@ void TestHash::CompareHashTimes_Small() const OUTPUT( "xxHash-64 : %2.3fs @ %6.3f GiB/s (hash: %016" PRIx64 ")\n", (double)time, (double)speed, crc ); } + // xxHash3 - 64 + { + const Timer t; + uint64_t crc( 0 ); + for ( size_t j = 0; j < numIterations; ++j ) + { + for ( size_t i = 0; i < numStrings; ++i ) + { + crc += xxHash3::Calc64( strings[ i ].Get(), strings[ i ].GetLength() ); + } + } + const float time = t.GetElapsed(); + const float speed = ( (float)dataSize / (float)( 1024 * 1024 * 1024 ) ) / time; + OUTPUT( "xxHash3-64 : %2.3fs @ %6.3f GiB/s (hash: %016" PRIx64 ")\n", (double)time, (double)speed, crc ); + } + // CRC32 - 8x8 slicing { const Timer t; diff --git a/Code/Core/CoreTest/Tests/TestMutex.cpp b/Code/Core/CoreTest/Tests/TestMutex.cpp index ab9d9f4ed..d1acfea9e 100644 --- a/Code/Core/CoreTest/Tests/TestMutex.cpp +++ b/Code/Core/CoreTest/Tests/TestMutex.cpp @@ -8,6 +8,7 @@ #include "Core/Mem/Mem.h" #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" +#include "Core/Process/Semaphore.h" #include "Core/Process/Thread.h" // TestMutex @@ -20,6 +21,11 @@ class TestMutex : public TestGroup void TestConstruct() const; void TestLockUnlock() const; void TestMultiLockUnlock() const; + void TryLock() const; + void TryLockMultiple() const; + void TryLockMixed() const; + void TryLockFail() const; + static uint32_t TestLockFailThreadEntryFunction( void * data ); void TestExclusivity() const; struct TestExclusivityUserData @@ -47,6 +53,10 @@ REGISTER_TESTS_BEGIN( TestMutex ) REGISTER_TEST( TestConstruct ) REGISTER_TEST( TestLockUnlock ) REGISTER_TEST( TestMultiLockUnlock ) + REGISTER_TEST( TryLock ) + REGISTER_TEST( TryLockMultiple ) + REGISTER_TEST( TryLockMixed ) + REGISTER_TEST( TryLockFail ) REGISTER_TEST( TestExclusivity ) #if defined( __WINDOWS__ ) REGISTER_TEST( TestAlignment ) @@ -80,6 +90,86 @@ void TestMutex::TestMultiLockUnlock() const m.Unlock(); } +// TryLock +//------------------------------------------------------------------------------ +void TestMutex::TryLock() const +{ + // Ensure lock can be acquired + Mutex m; + TEST_ASSERT( m.TryLock() ); + m.Unlock(); +} + +// TryLockMultiple +//------------------------------------------------------------------------------ +void TestMutex::TryLockMultiple() const +{ + // Ensure lock can be acquired multiple times + Mutex m; + { + // Manual locks + TEST_ASSERT( m.TryLock() ); + TEST_ASSERT( m.TryLock() ); + m.Unlock(); + m.Unlock(); + } + { + // RAII lock + MutexHolder mh( m ); + MutexHolder m2( m ); + } + { + // RAII lock (Try) + TryMutexHolder mh( m ); + TEST_ASSERT( mh.IsLocked() ); + TryMutexHolder mh2( m ); + TEST_ASSERT( mh2.IsLocked() ); + } +} + +// TryLockMixed +//------------------------------------------------------------------------------ +void TestMutex::TryLockMixed() const +{ + Mutex m; + // Lock then TryLock + { + m.Lock(); + TEST_ASSERT( m.TryLock() ); + m.Unlock(); + m.Unlock(); + } + // TryLock then Lock + { + TEST_ASSERT( m.TryLock() ); + m.Lock(); + m.Unlock(); + m.Unlock(); + } +} + +// TryLockFail +//------------------------------------------------------------------------------ +void TestMutex::TryLockFail() const +{ + Mutex m; + MutexHolder mh( m ); // Hold lock so thread cannot acquire it + + Thread t; + t.Start( TestLockFailThreadEntryFunction, "TryLockFail", &m ); + t.Join(); +} + +// TestLockFailThreadEntryFunction +//------------------------------------------------------------------------------ +/*static*/ uint32_t TestMutex::TestLockFailThreadEntryFunction( void * data ) +{ + // main thread should hold lock + Mutex * m = static_cast( data ); + TEST_ASSERT( m->TryLock() == false ); + return 0; +} + // TestExclusivity //------------------------------------------------------------------------------ void TestMutex::TestExclusivity() const @@ -88,10 +178,8 @@ void TestMutex::TestExclusivity() const data.m_Count = 0; data.m_BarrierCounter = 0; - Thread::ThreadHandle h = Thread::CreateThread( TestExclusivityThreadEntryFunction, - "TestExclusivity", - ( 64 * KILOBYTE ), - &data ); + Thread t; + t.Start( TestExclusivityThreadEntryFunction, "TestExclusivity", &data ); // arrive at barrier and wait AtomicInc( &data.m_BarrierCounter ); @@ -105,10 +193,7 @@ void TestMutex::TestExclusivity() const } // wait for other thread to complete - bool timedOut = false; - Thread::WaitForThread( h, 1000, timedOut ); - TEST_ASSERT( timedOut == false ); - Thread::CloseHandle( h ); + t.Join(); // ensure total is correct TEST_ASSERT( data.m_Count == 2000000 ); diff --git a/Code/Core/CoreTest/Tests/TestSemaphore.cpp b/Code/Core/CoreTest/Tests/TestSemaphore.cpp index 33175793b..c270ab50a 100644 --- a/Code/Core/CoreTest/Tests/TestSemaphore.cpp +++ b/Code/Core/CoreTest/Tests/TestSemaphore.cpp @@ -52,7 +52,8 @@ void TestSemaphore::WaitForSignal() const Semaphore s; // Create a thread which will signal the Semaphore - Thread::ThreadHandle h = Thread::CreateThread( WaitForSignal_Thread, "Test::WaitForSignal", ( 32 * KILOBYTE ), &s ); + Thread t; + t.Start( WaitForSignal_Thread, "Test::WaitForSignal", &s ); // Wait or the expected signal count for ( size_t i = 0; i < 100; ++i ) @@ -61,10 +62,7 @@ void TestSemaphore::WaitForSignal() const } // Cleanup thread - bool timedOut; - Thread::WaitForThread( h, 1000, timedOut ); - TEST_ASSERT( timedOut == false ); - Thread::CloseHandle( h ); + t.Join(); } // WaitForSignal_Thread diff --git a/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp b/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp index a1966c2c9..819af83fb 100644 --- a/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp +++ b/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp @@ -30,7 +30,7 @@ class TestSmallBlockAllocator : public TestGroup class ThreadInfo { public: - Thread::ThreadHandle m_ThreadHandle = INVALID_THREAD_HANDLE; + Thread m_Thread; Array< uint32_t > * m_AllocationSizes = nullptr; uint32_t m_RepeatCount = 0; float m_TimeTaken = 0.0f; @@ -101,17 +101,13 @@ void TestSmallBlockAllocator::MultiThreaded() const { info[ i ].m_AllocationSizes = & allocSizes; info[ i ].m_RepeatCount = repeatCount; - info[ i ].m_ThreadHandle = Thread::CreateThread( ThreadFunction_System, "SmallBlock", ( 64 * KILOBYTE ), (void*)&info[ i ] ); - TEST_ASSERT( info[ i ].m_ThreadHandle != INVALID_THREAD_HANDLE ); + info[ i ].m_Thread.Start( ThreadFunction_System, "SmallBlock", (void*)&info[ i ] ); } // Join the threads for ( size_t i = 0; i < numThreads; ++i ) { - bool timedOut; - Thread::WaitForThread( info[ i ].m_ThreadHandle, 500 * 1000, timedOut ); - Thread::CloseHandle( info[ i ].m_ThreadHandle ); - TEST_ASSERT( timedOut == false ); + info[ i ].m_Thread.Join(); time1 += info[ i ].m_TimeTaken; } } @@ -124,17 +120,13 @@ void TestSmallBlockAllocator::MultiThreaded() const { info[ i ].m_AllocationSizes = & allocSizes; info[ i ].m_RepeatCount = repeatCount; - info[ i ].m_ThreadHandle = Thread::CreateThread( ThreadFunction_SmallBlock, "SmallBlock", ( 64 * KILOBYTE ), (void*)&info[ i ] ); - TEST_ASSERT( info[ i ].m_ThreadHandle != INVALID_THREAD_HANDLE ); + info[ i ].m_Thread.Start( ThreadFunction_SmallBlock, "SmallBlock", (void*)&info[ i ] ); } // Join the threads for ( size_t i = 0; i < numThreads; ++i ) { - bool timedOut; - Thread::WaitForThread( info[ i ].m_ThreadHandle, 500 * 1000, timedOut ); - Thread::CloseHandle( info[ i ].m_ThreadHandle ); - TEST_ASSERT( timedOut == false ); + info[ i ].m_Thread.Join(); time2 += info[ i ].m_TimeTaken; } time2 /= numThreads; diff --git a/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp b/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp index bafce54f7..ac672c59c 100644 --- a/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp +++ b/Code/Core/CoreTest/Tests/TestTCPConnectionPool.cpp @@ -23,7 +23,7 @@ #define NUM_TEST_PASSES ( 16 ) // unique port for test in all configs so the tests can run in parallel -#ifdef WIN64 +#if defined( __WINDOWS__ ) #ifdef DEBUG #define TEST_PORT uint16_t( 21941 ) // arbitrarily chosen #else @@ -288,10 +288,8 @@ void TestTestTCPConnectionPool::TestConnectionStuckDuringSend() const TEST_ASSERT( ci ); // start a thread to flood the slow server - Thread::ThreadHandle h = Thread::CreateThread( TestConnectionStuckDuringSend_ThreadFunc, - "Sender", - ( 64 * KILOBYTE ), - (void *)ci ); + Thread thread; + thread.Start( TestConnectionStuckDuringSend_ThreadFunc, "Sender", (void *)ci ); // let thread send enough data to become blocked in Send Thread::Sleep( 100 ); @@ -301,8 +299,7 @@ void TestTestTCPConnectionPool::TestConnectionStuckDuringSend() const // wait for client thread to exit, with timeout bool timedOut( false ); - Thread::WaitForThread( h, 1000, timedOut ); - Thread::CloseHandle( h ); + thread.JoinWithTimeout( 1000, timedOut ); // TODO:B Remove use of unsafe API // if timeout was hit, things were stuck TEST_ASSERT( timedOut == false ); diff --git a/Code/Core/CoreTest/Tests/TestThread.cpp b/Code/Core/CoreTest/Tests/TestThread.cpp index ee14d7a3f..43451a3e0 100644 --- a/Code/Core/CoreTest/Tests/TestThread.cpp +++ b/Code/Core/CoreTest/Tests/TestThread.cpp @@ -41,6 +41,7 @@ void TestThread::Unused() const { // A thread object never used to create a thread Thread t; + TEST_ASSERT( t.IsRunning() == false ); } // StartAndJoin @@ -55,10 +56,12 @@ void TestThread::StartAndJoin() const t.Start( ThreadFunc, "StartAndJoin", reinterpret_cast( static_cast( userData ) ) ); + TEST_ASSERT( t.IsRunning() ); // Join and check result const uint32_t result = t.Join(); TEST_ASSERT( result == userData ); + TEST_ASSERT( t.IsRunning() == false ); } //------------------------------------------------------------------------------ diff --git a/Code/Core/CoreTest/Tests/TestTimer.cpp b/Code/Core/CoreTest/Tests/TestTimer.cpp index bca07f67c..272ca8461 100644 --- a/Code/Core/CoreTest/Tests/TestTimer.cpp +++ b/Code/Core/CoreTest/Tests/TestTimer.cpp @@ -32,7 +32,7 @@ void TestTimer::Validate() const Timer t; t.Start(); const int64_t before = t.GetNow(); - #if defined( __OSX__ ) && defined( __ARM64__ ) + #if defined( __OSX__ ) && defined( __aarch64__ ) // ARM // TODO:B Figure out why sleep granularity is so poor on Apple Silicon Thread::Sleep( 100 ); // sleep for 100ms #else diff --git a/Code/Core/Env/Types.h b/Code/Core/Env/Types.h index 19c8a3445..09775fa22 100644 --- a/Code/Core/Env/Types.h +++ b/Code/Core/Env/Types.h @@ -72,28 +72,19 @@ typedef signed int int32_t; #endif #ifndef intptr_t - #if defined( WIN64 ) + #if defined( __WINDOWS__ ) typedef int64_t intptr_t; typedef uint64_t uintptr_t; - #elif defined( WIN32 ) - typedef int32_t intptr_t; - typedef uint32_t uintptr_t; #endif #endif #ifndef uintptr_t #if defined( __LINUX__ ) - #if defined( __X64__ ) || defined( __ARM64__ ) - typedef uint64_t uintptr_t; - #else - typedef uint32_t uintptr_t; - #endif + typedef uint64_t uintptr_t; #endif #endif #ifndef size_t - #if defined( WIN64 ) + #if defined( __WINDOWS__ ) typedef uint64_t size_t; - #elif defined( WIN32 ) - typedef uint32_t size_t; #endif #endif @@ -120,7 +111,7 @@ typedef signed int int32_t; // Warning disabling //------------------------------------------------------------------------------ -#if defined( WIN32 ) || defined( WIN64 ) +#if defined( __WINDOWS__ ) #define PRAGMA_DISABLE_PUSH_MSVC( num ) __pragma(warning(push)) \ __pragma(warning(disable:num)) #define PRAGMA_DISABLE_POP_MSVC __pragma(warning(pop)) diff --git a/Code/Core/FileIO/FileIO.cpp b/Code/Core/FileIO/FileIO.cpp index 5addcfdb6..8cb5615d7 100644 --- a/Code/Core/FileIO/FileIO.cpp +++ b/Code/Core/FileIO/FileIO.cpp @@ -500,7 +500,7 @@ } #elif defined( __LINUX__ ) || defined( __APPLE__ ) umask( 0 ); // disable default creation mask // TODO:LINUX TODO:MAC Changes global program state; needs investigation - mode_t mode = S_IRWXU | S_IRWXG | S_IRWXO; // TODO:LINUX TODO:MAC Check these permissions + mode_t mode = ( S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH ); if ( mkdir( path.Get(), mode ) == 0 ) { return true; // created ok @@ -888,7 +888,8 @@ { return true; // can't even get the attributes, treat as not read only } - const bool currentlyReadOnly = !( ( s.st_mode & S_IWUSR ) == S_IWUSR ); // TODO:LINUX Is this the correct behaviour? + const uint32_t anyWriteBits = S_IWUSR | S_IWGRP | S_IWOTH; + const bool currentlyReadOnly = ( ( s.st_mode & anyWriteBits ) == 0 ); if ( readOnly == currentlyReadOnly ) { return true; // already in desired state @@ -897,13 +898,13 @@ // update writable flag if ( readOnly ) { - // remove writable flag - s.st_mode &= ( ~S_IWUSR ); // TODO:LINUX Is this the correct behaviour? + // remove writable flag for everyone + s.st_mode &= ~( S_IWUSR | S_IWGRP | S_IWOTH ); } else { - // add writable flag - s.st_mode |= S_IWUSR; // TODO:LINUX Is this the correct behaviour? + // add writable flag for just the user + s.st_mode |= S_IWUSR; } if ( chmod( fileName, s.st_mode ) == 0 ) @@ -948,11 +949,14 @@ #if defined( __LINUX__ ) || defined( __APPLE__ ) /*static*/ bool FileIO::SetExecutable( const char * fileName ) { - // rwxr-x--x (751) TODO:LINUX TODO:MAC Is this correct? - mode_t mode = S_IRUSR | S_IWUSR | S_IXUSR | - S_IRGRP | S_IXGRP | - S_IXOTH; - if ( chmod( fileName, mode ) == 0 ) + struct stat s; + if ( lstat( fileName, &s ) != 0 ) + { + return false; // failed to stat + } + + s.st_mode |= S_IXUSR | S_IXGRP | S_IXOTH; + if ( chmod( fileName, s.st_mode ) == 0 ) { return true; } diff --git a/Code/Core/Math/xxHash.h b/Code/Core/Math/xxHash.h index 0f494edc2..1dea11d68 100644 --- a/Code/Core/Math/xxHash.h +++ b/Code/Core/Math/xxHash.h @@ -10,8 +10,12 @@ // avoid including xxhash header directly extern "C" { + // xxHash unsigned int XXH32( const void * input, size_t length, unsigned seed ); unsigned long long XXH64( const void * input, size_t length, unsigned long long seed ); + + // xxhash3 + unsigned long long xxHashLib_XXH3_64bits( const void * input, size_t length ); }; // xxHash @@ -28,6 +32,16 @@ class xxHash enum { XXHASH_SEED = 0x0 }; // arbitrarily chosen random seed }; +// xxHash3 +//------------------------------------------------------------------------------ +class xxHash3 +{ +public: + inline static uint64_t Calc64( const void * buffer, size_t len ); + + inline static uint64_t Calc64( const AString & string ) { return Calc64( string.Get(), string.GetLength() ); } +}; + // Calc32 //------------------------------------------------------------------------------ /*static*/ uint32_t xxHash::Calc32( const void * buffer, size_t len ) @@ -42,4 +56,11 @@ class xxHash return XXH64( buffer, len, XXHASH_SEED ); } +// Calc64 (xxHash3) +//------------------------------------------------------------------------------ +/*static*/ uint64_t xxHash3::Calc64( const void * buffer, size_t len ) +{ + return xxHashLib_XXH3_64bits( buffer, len ); +} + //------------------------------------------------------------------------------ diff --git a/Code/Core/Network/Network.cpp b/Code/Core/Network/Network.cpp index 5be4d3e73..87913b648 100644 --- a/Code/Core/Network/Network.cpp +++ b/Code/Core/Network/Network.cpp @@ -173,19 +173,17 @@ data.safeToFree = false; // will be marked by other thread // Create thread to perform resolution - Thread::ThreadHandle handle = Thread::CreateThread( NameResolutionThreadFunc, - "NameResolution", - ( 32 * KILOBYTE ), - &data ); + Thread thread; + thread.Start( NameResolutionThreadFunc, "NameResolution", &data, ( 32 * KILOBYTE ) ); // wait for name resolution with timeout bool timedOut( false ); - int returnCode( 0 ); + uint32_t returnCode( 0 ); uint32_t remainingTimeMS( timeoutMS ); const uint32_t sleepInterval( 100 ); // Check exit condition periodically - TODO:C would be better to use an event for ( ;; ) { - returnCode = Thread::WaitForThread( handle, sleepInterval, timedOut ); + returnCode = thread.JoinWithTimeout( sleepInterval, timedOut ); // TODO:B Remove use of this unsafe API // Are we shutting down? if ( NetworkStartupHelper::IsShuttingDown() ) @@ -210,9 +208,8 @@ } if ( timedOut ) { - Thread::DetachThread( handle ); + thread.Detach(); // TODO:B Remove use of this unsafe API } - Thread::CloseHandle( handle ); // handle race where timeout occurred before thread marked data as // safe to delete (this could happen if system was under load and timeout was very small) @@ -227,7 +224,7 @@ } // return result of resolution (could also have failed) - return (uint32_t)returnCode; + return returnCode; } // NameResolutionThreadFunc diff --git a/Code/Core/Network/TCPConnectionPool.cpp b/Code/Core/Network/TCPConnectionPool.cpp index 04fb0dd79..2e9829c00 100644 --- a/Code/Core/Network/TCPConnectionPool.cpp +++ b/Code/Core/Network/TCPConnectionPool.cpp @@ -920,13 +920,9 @@ void TCPConnectionPool::CreateListenThread( TCPSocket socket, uint32_t host, uin m_ListenConnection->m_ThreadQuitNotification.Store( false ); // Spawn thread to handle socket - Thread::ThreadHandle h = Thread::CreateThread( &ListenThreadWrapperFunction, - "TCPListen", - ( 32 * KILOBYTE ), - m_ListenConnection ); // user data argument - ASSERT( h != INVALID_THREAD_HANDLE ); - Thread::DetachThread( h ); - Thread::CloseHandle( h ); // we don't need this anymore + Thread thread; + thread.Start( &ListenThreadWrapperFunction, "TCPListen", m_ListenConnection, ( 32 * KILOBYTE ) ); + thread.Detach(); // TODO: Remove use of this unsafe API } // ThreadWrapperFunction @@ -1043,13 +1039,9 @@ ConnectionInfo * TCPConnectionPool::CreateConnectionThread( TCPSocket socket, ui #endif // Spawn thread to handle socket - Thread::ThreadHandle h = Thread::CreateThread( &ConnectionThreadWrapperFunction, - "TCPConnection", - ( 64 * KILOBYTE ), - ci ); // user data argument - ASSERT( h != INVALID_THREAD_HANDLE ); - Thread::DetachThread( h ); - Thread::CloseHandle( h ); // we don't need this anymore + Thread thread; + thread.Start( &ConnectionThreadWrapperFunction, "TCPConnection", ci ); + thread.Detach(); // TODO:B Remove use of this unsafe API m_Connections.Append( ci ); diff --git a/Code/Core/Process/Mutex.cpp b/Code/Core/Process/Mutex.cpp index bc1047a6e..fbf853028 100644 --- a/Code/Core/Process/Mutex.cpp +++ b/Code/Core/Process/Mutex.cpp @@ -10,8 +10,7 @@ #if defined( __WINDOWS__ ) #include "Core/Env/WindowsHeader.h" -#endif -#if defined( __LINUX__ ) || defined( __APPLE__ ) +#else #include #endif @@ -24,13 +23,11 @@ Mutex::Mutex() static_assert( __alignof( decltype( m_CriticalSection ) ) == __alignof( CRITICAL_SECTION ), "Unexpected __alignof(CRITICAL_SECTION)" ); VERIFY( InitializeCriticalSectionAndSpinCount( (CRITICAL_SECTION *)&m_CriticalSection, 100 ) ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) + #else pthread_mutexattr_t attributes; VERIFY( pthread_mutexattr_init( &attributes ) == 0 ); pthread_mutexattr_settype( &attributes, PTHREAD_MUTEX_RECURSIVE ); VERIFY( pthread_mutex_init( &m_Mutex, &attributes ) == 0 ); - #else - #error Unknown platform #endif } @@ -40,10 +37,8 @@ Mutex::~Mutex() { #if defined( __WINDOWS__ ) DeleteCriticalSection( (CRITICAL_SECTION *)&m_CriticalSection ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) - VERIFY( pthread_mutex_destroy( &m_Mutex ) == 0 ); #else - #error Unknown platform + VERIFY( pthread_mutex_destroy( &m_Mutex ) == 0 ); #endif } @@ -54,22 +49,30 @@ void Mutex::Lock() { #if defined( __WINDOWS__ ) EnterCriticalSection( (CRITICAL_SECTION *)&m_CriticalSection ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) + #else VERIFY( pthread_mutex_lock( &m_Mutex ) == 0 ); + #endif +} + +// TryLock +//------------------------------------------------------------------------------ +bool Mutex::TryLock() +{ + #if defined( __WINDOWS__ ) + return ( TryEnterCriticalSection( (CRITICAL_SECTION *)&m_CriticalSection ) != FALSE ); #else - #error Unknown platform + return ( pthread_mutex_trylock( &m_Mutex ) == 0 ); #endif } + // Unlock //------------------------------------------------------------------------------ void Mutex::Unlock() { #if defined( __WINDOWS__ ) LeaveCriticalSection( (CRITICAL_SECTION *)&m_CriticalSection ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) - VERIFY( pthread_mutex_unlock( &m_Mutex ) == 0 ); #else - #error Unknown platform + VERIFY( pthread_mutex_unlock( &m_Mutex ) == 0 ); #endif } PRAGMA_DISABLE_POP_MSVC diff --git a/Code/Core/Process/Mutex.h b/Code/Core/Process/Mutex.h index 4d26573c7..ceed04a59 100644 --- a/Code/Core/Process/Mutex.h +++ b/Code/Core/Process/Mutex.h @@ -18,18 +18,15 @@ class Mutex Mutex(); ~Mutex(); - void Lock(); - void Unlock(); + void Lock(); + [[nodiscard]] bool TryLock(); + void Unlock(); private: // do this to avoid including windows.h - #if defined ( WIN64 ) + #if defined( __WINDOWS__ ) uint64_t m_CriticalSection[ 5 ]; // CRITICAL_SECTION - #elif defined ( WIN32 ) - uint32_t m_CriticalSection[ 6 ]; // CRITICAL_SECTION - #endif - - #if defined( __LINUX__ ) || defined( __APPLE__ ) + #else pthread_mutex_t m_Mutex; #endif }; @@ -55,4 +52,32 @@ class MutexHolder Mutex & m_Mutex; }; +// TryMutexHolder +//------------------------------------------------------------------------------ +class TryMutexHolder +{ +public: + explicit TryMutexHolder( Mutex & mutex ) + : m_Mutex( mutex ) + , m_Locked( mutex.TryLock() ) + { + } + ~TryMutexHolder() + { + if ( m_Locked ) + { + m_Mutex.Unlock(); + } + } + + [[nodiscard]] bool IsLocked() const { return m_Locked; } + +private: + TryMutexHolder( const TryMutexHolder & other ) = delete; + void operator = ( TryMutexHolder & other ) = delete; + + Mutex & m_Mutex; + const bool m_Locked; +}; + //------------------------------------------------------------------------------ diff --git a/Code/Core/Process/Thread.cpp b/Code/Core/Process/Thread.cpp index 165df23de..76af89591 100644 --- a/Code/Core/Process/Thread.cpp +++ b/Code/Core/Process/Thread.cpp @@ -111,11 +111,47 @@ void Thread::Start( ThreadEntryFunction func, uint32_t stackSizeBytes ) { // Can only start if not already started - ASSERT( m_Handle == INVALID_THREAD_HANDLE ); + ASSERT( !IsRunning() ); - // Start thread - m_Handle = CreateThread( func, threadName, stackSizeBytes, userData ); - ASSERT( m_Handle != INVALID_THREAD_HANDLE ); + // Create structure to pass to thread + ThreadStartInfo & info = *FNEW( ThreadStartInfo( func, userData, threadName ) ); + MemoryBarrier(); + + // Create thread + #if defined( __WINDOWS__ ) + HANDLE h = ::CreateThread( nullptr, // LPSECURITY_ATTRIBUTES lpThreadAttributes + stackSizeBytes, // SIZE_T dwStackSize + (LPTHREAD_START_ROUTINE)ThreadStartInfo::ThreadStartFunction, // LPTHREAD_START_ROUTINE lpStartAddress + &info, // LPVOID lpParameter + 0, // DWORD dwCreationFlags + nullptr // LPDWORD lpThreadId + ); + ASSERT( h != nullptr ); + #elif defined( __LINUX__ ) || defined( __APPLE__ ) + #if __has_feature( address_sanitizer ) || defined( __SANITIZE_ADDRESS__ ) + // AddressSanitizer instruments objects created on the stack by inserting redzones around them. + // This greatly increases the amount of stack space used by the program. + // To account for that double the requested stack size for the thread. + stackSizeBytes *= 2; + #endif + // Necessary on Aarch64, where it's 131072 in my tests. Sometimes we ask for 65536. + if ( stackSizeBytes < PTHREAD_STACK_MIN ) + { + stackSizeBytes = PTHREAD_STACK_MIN; + } + pthread_t h( 0 ); + pthread_attr_t threadAttr; + VERIFY( pthread_attr_init( &threadAttr ) == 0 ); + VERIFY( pthread_attr_setstacksize( &threadAttr, stackSizeBytes ) == 0 ); + VERIFY( pthread_attr_setdetachstate( &threadAttr, PTHREAD_CREATE_JOINABLE ) == 0 ); + VERIFY( pthread_create( &h, &threadAttr, ThreadStartInfo::ThreadStartFunction, &info ) == 0 ); + ASSERT( h != (pthread_t)0 ); + #else + #error Unknown platform + #endif + m_Handle = (Thread::ThreadHandle)h; + + ASSERT( IsRunning() ); } // Join @@ -123,17 +159,39 @@ void Thread::Start( ThreadEntryFunction func, uint32_t Thread::Join() { // Must only join if running and not already joined - ASSERT( m_Handle != INVALID_THREAD_HANDLE ); + ASSERT( IsRunning() ); // Wait for thread and obtain return result - // TODO:C Fix inconsistent return results when legacy API is removed - const uint32_t returnValue = static_cast( WaitForThread( m_Handle ) ); - - // Clean up the handle - CloseHandle( m_Handle ); - m_Handle = INVALID_THREAD_HANDLE; + #if defined( __WINDOWS__ ) + // Wait for thread to finish + VERIFY( WaitForSingleObject( m_Handle, INFINITE ) == WAIT_OBJECT_0 ); + + // Ge teturn code + DWORD ret = 0; + VERIFY( ::GetExitCodeThread( m_Handle, (LPDWORD)&ret ) ); + VERIFY( ::CloseHandle( m_Handle ) ); + m_Handle = INVALID_THREAD_HANDLE; + return ret; + #elif defined( __APPLE__ ) || defined( __LINUX__ ) + void * ret; + if ( pthread_join( (pthread_t)m_Handle, &ret ) == 0 ) + { + m_Handle = INVALID_THREAD_HANDLE; + return static_cast( (size_t)ret ); + } + ASSERT( false ); // usage failure + m_Handle = INVALID_THREAD_HANDLE; + return 0; + #else + #error Unknown platform + #endif +} - return returnValue; +// IsRunning +//------------------------------------------------------------------------------ +bool Thread::IsRunning() const +{ + return ( m_Handle != INVALID_THREAD_HANDLE ); } // GetCurrentThreadId @@ -155,7 +213,7 @@ uint32_t Thread::Join() { PROFILE_FUNCTION; - #if defined( WIN32 ) || defined( WIN64 ) + #if defined( __WINDOWS__ ) ::Sleep( ms ); #elif defined( __APPLE__ ) || defined( __LINUX__ ) usleep( ms * 1000 ); @@ -164,121 +222,60 @@ uint32_t Thread::Join() #endif } -// CreateThread +// Detach //------------------------------------------------------------------------------ -/*static*/ Thread::ThreadHandle Thread::CreateThread( ThreadEntryFunction entryFunc, - const char * threadName, - uint32_t stackSize, - void * userData ) +void Thread::Detach() { - ThreadStartInfo & info = *FNEW( ThreadStartInfo( entryFunc, userData, threadName ) ); - MemoryBarrier(); + // TODO:B Remove this unsafe function - #if defined( __WINDOWS__ ) - HANDLE h = ::CreateThread( nullptr, // LPSECURITY_ATTRIBUTES lpThreadAttributes - stackSize, // SIZE_T dwStackSize - (LPTHREAD_START_ROUTINE)ThreadStartInfo::ThreadStartFunction, // LPTHREAD_START_ROUTINE lpStartAddress - &info, // LPVOID lpParameter - 0, // DWORD dwCreationFlags - nullptr // LPDWORD lpThreadId - ); - ASSERT( h != nullptr ); - #elif defined( __LINUX__ ) || defined( __APPLE__ ) - #if __has_feature( address_sanitizer ) || defined( __SANITIZE_ADDRESS__ ) - // AddressSanitizer instruments objects created on the stack by inserting redzones around them. - // This greatly increases the amount of stack space used by the program. - // To account for that double the requested stack size for the thread. - stackSize *= 2; - #endif - // Necessary on Aarch64, where it's 131072 in my tests. Sometimes we ask for 65536. - if ( stackSize < PTHREAD_STACK_MIN ) - { - stackSize = PTHREAD_STACK_MIN; - } - pthread_t h( 0 ); - pthread_attr_t threadAttr; - VERIFY( pthread_attr_init( &threadAttr ) == 0 ); - VERIFY( pthread_attr_setstacksize( &threadAttr, stackSize ) == 0 ); - VERIFY( pthread_attr_setdetachstate( &threadAttr, PTHREAD_CREATE_JOINABLE ) == 0 ); - VERIFY( pthread_create( &h, &threadAttr, ThreadStartInfo::ThreadStartFunction, &info ) == 0 ); - ASSERT( h != (pthread_t)0 ); - #else - #error Unknown platform - #endif + // Must only detach if running and not already joined or detached + ASSERT( IsRunning() ); - return (Thread::ThreadHandle)h; -} - -// WaitForThread -//------------------------------------------------------------------------------ -/*static*/ int32_t Thread::WaitForThread( ThreadHandle handle ) -{ #if defined( __WINDOWS__ ) - bool timedOut = true; // default is true to catch cases when timedOut wasn't set by WaitForThread() - const int32_t ret = WaitForThread( handle, INFINITE, timedOut ); - - if ( timedOut ) - { - // something is wrong - we were waiting an INFINITE time - ASSERT( false ); - return 0; - } - - return ret; + VERIFY( ::CloseHandle( m_Handle ) ); #elif defined( __APPLE__ ) || defined( __LINUX__ ) - void * ret; - if ( pthread_join( (pthread_t)handle, &ret ) == 0 ) - { - return (int)( (size_t)ret ); - } - ASSERT( false ); // usage failure - return 0; + VERIFY( pthread_detach( (pthread_t)m_Handle ) == 0 ); #else #error Unknown platform #endif + + m_Handle = INVALID_THREAD_HANDLE; } -// WaitForThread +// JoinWithTimeout //------------------------------------------------------------------------------ -/*static*/ int32_t Thread::WaitForThread( ThreadHandle handle, uint32_t timeoutMS, bool & timedOut ) +uint32_t Thread::JoinWithTimeout( uint32_t timeoutMS, bool & outTimedOut ) { + // TODO:B Remove this unsafe function + #if defined( __WINDOWS__ ) - // wait for thread to finish - const DWORD waitResult = WaitForSingleObject( handle, timeoutMS ); + // Wait for thread to finish + const DWORD waitResult = WaitForSingleObject( m_Handle, timeoutMS ); - // timed out ? + // Timed out? if ( waitResult == WAIT_TIMEOUT ) { - timedOut = true; + outTimedOut = true; return 0; } - if ( waitResult != WAIT_OBJECT_0 ) - { - // something is wrong - invalid handle? - ASSERT( false ); - timedOut = false; - return 0; - } - - // get actual return code - DWORD ret( 0 ); - if ( ::GetExitCodeThread( handle, (LPDWORD)&ret ) ) - { - timedOut = false; - return (int32_t)ret; - } - ASSERT( false ); // invalid thread handle - timedOut = false; - return -1; + VERIFY( waitResult == WAIT_OBJECT_0 ); // Invalid handle? + outTimedOut = false; + + // Get return code + DWORD ret = 0; + VERIFY( ::GetExitCodeThread( m_Handle, (LPDWORD)&ret ) ); + VERIFY( ::CloseHandle( m_Handle ) ); + m_Handle = INVALID_THREAD_HANDLE; + return ret; #elif __has_feature( thread_sanitizer ) || defined( __SANITIZE_THREAD__ ) // ThreadSanitizer doesn't support pthread_timedjoin_np yet (as of February 2018) - timedOut = false; + outTimedOut = false; (void)timeoutMS; - return WaitForThread( handle ); + return Join(); #elif defined( __APPLE__ ) - timedOut = false; + outTimedOut = false; (void)timeoutMS; // TODO:MAC Implement timeout support - return WaitForThread( handle ); + return Join(); #elif defined( __LINUX__ ) // timeout is specified in absolute time // - get current time @@ -298,47 +295,16 @@ uint32_t Thread::Join() // join thread with timeout void * ret; - int retVal = pthread_timedjoin_np( (pthread_t)handle, &ret, &abstime ); + int retVal = pthread_timedjoin_np( (pthread_t)m_Handle, &ret, &abstime ); if ( ( retVal == EBUSY ) || ( retVal == ETIMEDOUT ) ) { - timedOut = true; + outTimedOut = true; return 0; } - if ( retVal == 0 ) - { - timedOut = false; - return (int)( (size_t)ret ); - } - - ASSERT( false ); // a non-timeout error indicates usage failure - timedOut = false; - return 0; - #else - #error Unknown platform - #endif -} - -// DetachThread -//------------------------------------------------------------------------------ -/*static*/ void Thread::DetachThread( ThreadHandle handle ) -{ - #if defined( __WINDOWS__ ) - (void)handle; // Nothing to do - #elif defined( __APPLE__ ) || defined( __LINUX__ ) - VERIFY( pthread_detach( (pthread_t)handle ) == 0 ); - #else - #error Unknown platform - #endif -} - -// CloseHandle -//------------------------------------------------------------------------------ -/*static*/ void Thread::CloseHandle( ThreadHandle h ) -{ - #if defined( __WINDOWS__ ) - ::CloseHandle( h ); - #elif defined( __APPLE__ ) || defined( __LINUX__ ) - (void)h; // Nothing to do + VERIFY( retVal == 0 ); + m_Handle = INVALID_THREAD_HANDLE; + outTimedOut = false; + return static_cast( (size_t)ret ); #else #error Unknown platform #endif diff --git a/Code/Core/Process/Thread.h b/Code/Core/Process/Thread.h index dff648651..c62edfe43 100644 --- a/Code/Core/Process/Thread.h +++ b/Code/Core/Process/Thread.h @@ -41,6 +41,7 @@ class Thread void * userData = nullptr, uint32_t stackSizeBytes = kDefaultStackSize ); uint32_t Join(); + bool IsRunning() const; // Thread Identification static ThreadId GetCurrentThreadId(); @@ -51,16 +52,9 @@ class Thread // Sleeps static void Sleep( uint32_t ms ); - // Thread lifetime (Legacy API) - static ThreadHandle CreateThread( ThreadEntryFunction entryFunc, - const char * threadName = nullptr, - uint32_t stackSize = kDefaultStackSize, - void * userData = nullptr - ); - static int32_t WaitForThread( ThreadHandle handle ); - static int32_t WaitForThread( ThreadHandle handle, uint32_t timeoutMS, bool & timedOut ); - static void DetachThread( ThreadHandle handle ); - static void CloseHandle( ThreadHandle h ); + // Legacy Functions - TODO:B Remove these unsafe functions + void Detach(); // TODO:B Remove this unsafe function + uint32_t JoinWithTimeout( uint32_t timeoutMS, bool & outTimedOut ); // TODO:B Remove this unsafe API // Debugging static void SetThreadName( const char * name ); diff --git a/Code/Core/Time/Timer.cpp b/Code/Core/Time/Timer.cpp index d5c4a4f1c..557d3363c 100644 --- a/Code/Core/Time/Timer.cpp +++ b/Code/Core/Time/Timer.cpp @@ -38,7 +38,7 @@ class GlobalTimerFrequencyInitializer Timer::s_Frequency = freq.QuadPart; #endif #if defined( __APPLE__ ) - #if defined( __ARM64__ ) + #if defined( __aarch64__ ) // ARM Timer::s_Frequency = 1000000000; #else mach_timebase_info_data_t info; @@ -64,7 +64,7 @@ int64_t Timer::GetNow() VERIFY( QueryPerformanceCounter( &now ) ); return now.QuadPart; #elif defined( __APPLE__ ) - #if defined( __ARM64__ ) + #if defined( __aarch64__ ) // ARM // mach_absolute_time seems to return the wrong time on Apple Silicon return (int64_t)clock_gettime_nsec_np( CLOCK_MONOTONIC ); #else diff --git a/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff b/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff index 32609aaf9..b44292908 100644 --- a/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff +++ b/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff @@ -52,6 +52,7 @@ 'FBuildCore-Lib-$Platform$-$BuildConfigName$', 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/bfffuzzer$ExeExtension$' .LinkerOptions + ' -pthread -ldl -lrt' diff --git a/Code/Tools/FBuild/Documentation/docs/changelog.html b/Code/Tools/FBuild/Documentation/docs/changelog.html index e8aed1bd8..2486d3c9f 100644 --- a/Code/Tools/FBuild/Documentation/docs/changelog.html +++ b/Code/Tools/FBuild/Documentation/docs/changelog.html @@ -26,6 +26,45 @@

Changelog

+
+ v1.10 (28th-May-2023) +
+
+ Notes +
    +
  • DB Version Changed : clean build will occur
    • +
  • Cache version changed : cache will be re-populated
    • +
+ New +
    +
  • [OSX] Apple Silicon is supported natively using Universal binaries
    • +
+ Fixes +
    +
  • Distributed Compilation
    • +
      +
    • Prevent Worker over-requesting jobs when there are multiple Clients
      • +
    +
  • [Linux][OSX] Fix file/folder permissions being exessively permissive (Thanks to Pete Lewis)
    • +
  • Chained ObjectLists correctly handle duplicate names in sub-directories (Thanks to Dandielo)
    • +
  • Fix crash when .CompilerOutputPath is missing but required on ObjectList/Library
    • +
+ Improvements +
    +
  • Distributed Compilation
    • +
      +
    • Toolchain management performance improved slightly when toolchain changes
      • +
    • Remove some unecessary lock contention in FBuildWorker
      • +
    • FBuildWorker can be set to restart every 4 hours (-periodicrestart)
      • +
    • Eliminate various instances of lock contention in FBuildWorker
      • +
    +
  • Cache performance improved slightly for check, store and retrieve operations
    • +
  • Performance of various build steps improved slightly
    • +
  • Database validation improved, reducing startup/shutdown time slightly
    • +
  • Fix documenation errors for some optional fields (Thanks to Harrison Ting)
    • +
+
+
v1.09 (5th-Mar-2023)
diff --git a/Code/Tools/FBuild/Documentation/docs/download.html b/Code/Tools/FBuild/Documentation/docs/download.html index 6a1afa7d7..ad96b027d 100644 --- a/Code/Tools/FBuild/Documentation/docs/download.html +++ b/Code/Tools/FBuild/Documentation/docs/download.html @@ -31,28 +31,30 @@

Download

- Current Version - v1.09 + Current Version

See the changelog for a detailed list of changes.

-Binaries - -Source - +
+
VersionWindowsOS XLinuxSourceMarkup
v1.09x64x64x64Zip | + GitHubNotePad++Visual StudioBash completion
v1.08 x64 x64
+ + + + + + + +
VersionWindowsOS XLinuxSource
v1.10x64x64 & ARMx64Zip | + GitHub
+
+ Syntax Highlighting @@ -73,8 +75,9 @@

Unreal Engine

Editor Integration

    -
  • vim-fastbuild - Syntax highlighting for vim by dummyunit
    • -
  • FASTBuild-Sublime - Syntax highlighting for Sublime Text 3 by Manuzor
    • +
  • Sublime - Syntax highlighting for Sublime Text 3 by Manuzor
    • +
  • vim - Syntax highlighting for vim by dummyunit
    • +
  • VSCode - Syntax highlighting, go-to definition, find references, hover to see variable values, and much more by Harrison Ting
@@ -84,6 +87,16 @@

Editor Integration

+ + + + + + + + + diff --git a/Code/Tools/FBuild/Documentation/docs/functions/dll.html b/Code/Tools/FBuild/Documentation/docs/functions/dll.html index 8627b39b5..2347d0cd6 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/dll.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/dll.html @@ -38,7 +38,7 @@

DLL

.LinkerOutput ; Output from linker .LinkerOptions ; Options to pass to linker .Libraries ; Libraries to link into DLL - .Libraries2 ; Secondary libraries to link into executable + .Libraries2 ; (optional) Secondary libraries to link into executable .LinkerLinkObjects ; (optional) Link objects used to make libs instead of libs (default true) .LinkerAssemblyResources ; (optional) List of assembly resources to use with %3 diff --git a/Code/Tools/FBuild/Documentation/docs/functions/executable.html b/Code/Tools/FBuild/Documentation/docs/functions/executable.html index c51579e3e..70fc83ef8 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/executable.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/executable.html @@ -38,7 +38,7 @@

Executable

.LinkerOutput ; Output from linker .LinkerOptions ; Options to pass to linker .Libraries ; Libraries to link into executable - .Libraries2 ; Secondary libraries to link into executable + .Libraries2 ; (optional) Secondary libraries to link into executable .LinkerLinkObjects ; (optional) Link objects used to make libs instead of libs (default false) .LinkerAssemblyResources ; (optional) List of assembly resources to use with %3 diff --git a/Code/Tools/FBuild/Documentation/docs/functions/xcodeproject.html b/Code/Tools/FBuild/Documentation/docs/functions/xcodeproject.html index e72e5a225..da84218c6 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/xcodeproject.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/xcodeproject.html @@ -297,7 +297,7 @@

XCodeProject

-

.XCodeOrganizationName - String - (Required)

+

.XCodeOrganizationName - String - (Optional)

The organization name which appears in the generated project can be set.

Example:
.XCodeOrganizationName = 'MyCompany'
diff --git a/Code/Tools/FBuild/Documentation/docs/home.html b/Code/Tools/FBuild/Documentation/docs/home.html index 98efeaa09..e766d5477 100644 --- a/Code/Tools/FBuild/Documentation/docs/home.html +++ b/Code/Tools/FBuild/Documentation/docs/home.html @@ -38,12 +38,14 @@
- FASTBuild v1.09 released!  (5-March-2023) + FASTBuild v1.10 released!  (28-May-2023)
An updated version of FASTBuild can now be downloaded, featuring:
    -
  • Several bug fixes, reliability improvements and performance enhancements
    • +
  • Native support for Apple Silicon (ARM) Macs, significantly improving performance
    • +
  • Various performance enhancements
    • +
  • Some misc bug fixes and reliability improvements
  • Details...
diff --git a/Code/Tools/FBuild/Documentation/docs/options.html b/Code/Tools/FBuild/Documentation/docs/options.html index f84ca135f..3440cda31 100644 --- a/Code/Tools/FBuild/Documentation/docs/options.html +++ b/Code/Tools/FBuild/Documentation/docs/options.html @@ -243,6 +243,10 @@

FBuildWorker.exe

+ + + +
VersionWindowsOS XLinuxSourceMarkup
v1.09x64x64x64Zip | + GitHubNotePad++Visual StudioBash completion
v1.08 x64 x64 -nosubprocess Don't spawn as a sub-process.
-periodicrestartRestart worker every 4 hours.
@@ -718,6 +722,13 @@

FBuildWorker.exe Detailed

The "-nosubprocess" option suppresses this behaviour.

+
-periodicrestart
+
+

Restart worker every 4 hours.

+

If worker reliability issues are encountered, perhaps due to uncontrolable factors such as OS instability, network driver issues or as yet unresolved FASTBuild bugs, the worker can be instructed to periodically restart itself as a potential workaround.

+
+ +
© 2012-2023 Franta Fulin
diff --git a/Code/Tools/FBuild/FBuild/FBuild.bff b/Code/Tools/FBuild/FBuild/FBuild.bff index d67691e85..f3c260a6c 100644 --- a/Code/Tools/FBuild/FBuild/FBuild.bff +++ b/Code/Tools/FBuild/FBuild/FBuild.bff @@ -49,6 +49,7 @@ 'FBuildCore-Lib-$Platform$-$BuildConfigName$', 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } #if __LINUX__ .LinkerOutput = '$OutputBase$/$ProjectPath$/fbuild$ExeExtension$' // NOTE: lower case @@ -93,6 +94,19 @@ #endif } + // Create Universal binaries + #if CLANG_SUPPORTS_ARMOSX + ForEach( .BuildConfig in .BuildConfigs ) + { + Using( .BuildConfig ) + If( .Platform == 'ARMOSX' ) + { + CreateUniversalBinary( .ProjectName, .ProjectPath, .BuildConfigName, .OutputBase ) + ^'Targets_OSX_$BuildConfigName$' + { '$ProjectName$-OSX-$BuildConfigName$' } + } + } + #endif + // Aliases //-------------------------------------------------------------------------- CreateCommonAliases( .ProjectName ) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp index 21522c78b..2a0503c14 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFFile.cpp @@ -25,7 +25,7 @@ BFFFile::BFFFile( const char * fileName, const AString & fileContents ) , m_FileContents( fileContents ) , m_Once( false ) { - m_Hash = xxHash::Calc64( m_FileContents ); + m_Hash = xxHash3::Calc64( m_FileContents ); } // DESTRUCTOR @@ -68,7 +68,7 @@ bool BFFFile::Load( const AString & fileName, const BFFToken * token ) m_FileContents = Move( fileContents ); m_FileName = fileName; m_ModTime = FileIO::GetFileLastWriteTime( fileName ); - m_Hash = xxHash::Calc64( m_FileContents ); + m_Hash = xxHash3::Calc64( m_FileContents ); return true; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp index 076f2664e..4c786348d 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFMacros.cpp @@ -50,13 +50,13 @@ bool BFFMacros::IsDefined( const AString & token ) const return true; } #endif - #if defined( __X64__ ) + #if defined( __x86_64__ ) || defined( _M_X64 ) // X64 if ( token == "__X64__" ) { return true; } #endif - #if defined( __ARM64__ ) + #if defined( __aarch64__ ) || defined( _M_ARM64 ) // ARM if ( token == "__ARM64__" ) { return true; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.cpp index 8f00853b2..b3dc50fb7 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.cpp @@ -389,7 +389,7 @@ bool BFFParser::ParseVariableDeclaration( BFFTokenRange & iter, const AString & { newVal = i; } - return StoreVariableInt( varName, newVal, frame ); + return StoreVariableInt( varName, opToken, newVal, frame ); } else if ( rhsToken->IsBooelan() ) { @@ -415,7 +415,7 @@ bool BFFParser::ParseVariableDeclaration( BFFTokenRange & iter, const AString & iter++; // Consume the rhs - return StoreVariableBool( varName, rhsToken->GetBoolean(), frame ); + return StoreVariableBool( varName, rhsToken, rhsToken->GetBoolean(), frame ); } else if ( rhsToken->IsVariable() ) { @@ -695,15 +695,15 @@ bool BFFParser::ParseUserFunctionCall( BFFTokenRange & iter, const BFFUserFuncti { return false; } - BFFStackFrame::SetVarString( argName, value, &frame ); + BFFStackFrame::SetVarString( argName, *arg, value, &frame ); } else if ( arg->IsBooelan() ) { - BFFStackFrame::SetVarBool( argName, arg->GetBoolean(), &frame ); + BFFStackFrame::SetVarBool( argName, *arg, arg->GetBoolean(), &frame ); } else if ( arg->IsNumber() ) { - BFFStackFrame::SetVarInt( argName, arg->GetValueInt(), &frame ); + BFFStackFrame::SetVarInt( argName, *arg, arg->GetValueInt(), &frame ); } else { @@ -734,7 +734,7 @@ bool BFFParser::ParseUserFunctionCall( BFFTokenRange & iter, const BFFUserFuncti } // Set in function frame with argument name - BFFStackFrame::SetVar( varSrc, argName, &frame ); + BFFStackFrame::SetVar( varSrc, *arg, argName, &frame ); } } @@ -871,7 +871,7 @@ bool BFFParser::StoreVariableString( const AString & name, finalValue.Replace( value.Get(), "" ); } - BFFStackFrame::SetVarString( name, finalValue, frame ); + BFFStackFrame::SetVarString( name, *opToken, finalValue, frame ); return true; } else if ( var->IsArrayOfStrings() || dstIsEmpty ) @@ -898,7 +898,7 @@ bool BFFParser::StoreVariableString( const AString & name, } } - BFFStackFrame::SetVarArrayOfStrings( name, finalValues, frame ); + BFFStackFrame::SetVarArrayOfStrings( name, *opToken, finalValues, frame ); return true; } else @@ -913,7 +913,7 @@ bool BFFParser::StoreVariableString( const AString & name, if ( ( var == nullptr ) || var->IsString() ) { // OK - asigning to a new variable or to a string - BFFStackFrame::SetVarString( name, value, frame ); + BFFStackFrame::SetVarString( name, *opToken, value, frame ); return true; } else if ( var->IsArrayOfStrings() || dstIsEmpty ) @@ -921,7 +921,7 @@ bool BFFParser::StoreVariableString( const AString & name, // OK - store new string as the single element of array StackArray values; values.Append( value ); - BFFStackFrame::SetVarArrayOfStrings( name, values, frame ); + BFFStackFrame::SetVarArrayOfStrings( name, *opToken, values, frame ); return true; } else @@ -1171,13 +1171,13 @@ bool BFFParser::StoreVariableArray( const AString & name, if ( varType == BFFVariable::VAR_ARRAY_OF_STRUCTS ) { // structs - BFFStackFrame::SetVarArrayOfStructs( name, structValues, frame ); + BFFStackFrame::SetVarArrayOfStructs( name, *opToken, structValues, frame ); } else { ASSERT( varType == BFFVariable::VAR_ARRAY_OF_STRINGS ); // strings - BFFStackFrame::SetVarArrayOfStrings( name, values, frame ); + BFFStackFrame::SetVarArrayOfStrings( name, *opToken, values, frame ); } return true; @@ -1227,26 +1227,26 @@ bool BFFParser::StoreVariableStruct( const AString & name, Array & structMembers = stackFrame.GetLocalVariables(); // Register this variable - BFFStackFrame::SetVarStruct( name, Move( structMembers ), frame ? frame : stackFrame.GetParent() ); + BFFStackFrame::SetVarStruct( name, *operatorToken, Move( structMembers ), frame ? frame : stackFrame.GetParent() ); return true; } // StoreVariableBool //------------------------------------------------------------------------------ -bool BFFParser::StoreVariableBool( const AString & name, bool value, BFFStackFrame * frame ) +bool BFFParser::StoreVariableBool( const AString & name, const BFFToken * token, bool value, BFFStackFrame * frame ) { // Register this variable - BFFStackFrame::SetVarBool( name, value, frame ); + BFFStackFrame::SetVarBool( name, *token, value, frame ); return true; } // StoreVariableInt //------------------------------------------------------------------------------ -bool BFFParser::StoreVariableInt( const AString & name, int value, BFFStackFrame * frame ) +bool BFFParser::StoreVariableInt( const AString & name, const BFFToken * token, int value, BFFStackFrame * frame ) { - BFFStackFrame::SetVarInt( name, value, frame ); + BFFStackFrame::SetVarInt( name, *token, value, frame ); return true; } @@ -1372,7 +1372,7 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken values.Append( varSrc->GetString() ); } - BFFStackFrame::SetVarArrayOfStrings( dstName, values, dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), values, dstFrame ); return true; } @@ -1390,7 +1390,7 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken } values.Append( varSrc ); - BFFStackFrame::SetVarArrayOfStructs( dstName, values, dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), values, dstFrame ); return true; } @@ -1403,12 +1403,12 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken const BFFVariable * var = ( dstFrame ? dstFrame : BFFStackFrame::GetCurrent() )->GetLocalVar( dstName ); if ( varDst != var ) { - BFFStackFrame::SetVarArrayOfStrings( dstName, varDst->GetArrayOfStrings(), dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), varDst->GetArrayOfStrings(), dstFrame ); } } else { - BFFStackFrame::SetVarArrayOfStrings( dstName, Array< AString >(), dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), Array< AString >(), dstFrame ); } return true; } @@ -1422,12 +1422,12 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken const BFFVariable * var = ( dstFrame ? dstFrame : BFFStackFrame::GetCurrent() )->GetLocalVar( dstName ); if ( varDst != var ) { - BFFStackFrame::SetVarArrayOfStructs( dstName, varDst->GetArrayOfStructs(), dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), varDst->GetArrayOfStructs(), dstFrame ); } } else { - BFFStackFrame::SetVarArrayOfStructs(dstName, Array< const BFFVariable * >(), dstFrame); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), Array< const BFFVariable * >(), dstFrame ); } return true; } @@ -1435,14 +1435,14 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken // ArrayOfStrings to empty array, assignment or concatenation if ( dstIsEmpty && srcType == BFFVariable::VAR_ARRAY_OF_STRINGS && !subtract ) { - BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetArrayOfStrings(), dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), varSrc->GetArrayOfStrings(), dstFrame ); return true; } // ArrayOfStructs to empty array, assignment or concatenation if ( dstIsEmpty && srcType == BFFVariable::VAR_ARRAY_OF_STRUCTS && !subtract ) { - BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetArrayOfStructs(), dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), varSrc->GetArrayOfStructs(), dstFrame ); return true; } } @@ -1456,17 +1456,17 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken { AStackString< 2048 > finalValue(varDst->GetString()); finalValue += varSrc->GetString(); - BFFStackFrame::SetVarString( dstName, finalValue, dstFrame ); + BFFStackFrame::SetVarString( dstName, varSrc->GetToken(), finalValue, dstFrame ); } else if ( subtract ) { AStackString< 2048 > finalValue(varDst->GetString()); finalValue.Replace( varSrc->GetString().Get(), "" ); - BFFStackFrame::SetVarString( dstName, finalValue, dstFrame ); + BFFStackFrame::SetVarString( dstName, varSrc->GetToken(), finalValue, dstFrame ); } else { - BFFStackFrame::SetVarString( dstName, varSrc->GetString(), dstFrame ); + BFFStackFrame::SetVarString( dstName, varSrc->GetToken(), varSrc->GetString(), dstFrame ); } return true; } @@ -1480,11 +1480,11 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken values.SetCapacity( num ); values.Append( varDst->GetArrayOfStrings() ); values.Append( varSrc->GetArrayOfStrings() ); - BFFStackFrame::SetVarArrayOfStrings( dstName, values, dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), values, dstFrame ); } else { - BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetArrayOfStrings(), dstFrame ); + BFFStackFrame::SetVarArrayOfStrings( dstName, varSrc->GetToken(), varSrc->GetArrayOfStrings(), dstFrame ); } return true; } @@ -1498,11 +1498,11 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken values.SetCapacity( num ); values.Append( varDst->GetArrayOfStructs() ); values.Append( varSrc->GetArrayOfStructs() ); - BFFStackFrame::SetVarArrayOfStructs( dstName, values, dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), values, dstFrame ); } else { - BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetArrayOfStructs(), dstFrame ); + BFFStackFrame::SetVarArrayOfStructs( dstName, varSrc->GetToken(), varSrc->GetArrayOfStructs(), dstFrame ); } return true; } @@ -1522,12 +1522,12 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken { newVal = varSrc->GetInt(); } - return StoreVariableInt( dstName, newVal, dstFrame ); + return StoreVariableInt( dstName, &varSrc->GetToken(), newVal, dstFrame ); } if ( ( srcType == BFFVariable::VAR_BOOL ) && !concat && !subtract ) { - return StoreVariableBool( dstName, varSrc->GetBool(), dstFrame ); + return StoreVariableBool( dstName, &varSrc->GetToken(), varSrc->GetBool(), dstFrame ); } if ( ( srcType == BFFVariable::VAR_STRUCT ) && !subtract ) @@ -1544,7 +1544,7 @@ bool BFFParser::StoreVariableToVariable( const AString & dstName, const BFFToken else { // Register this variable - BFFStackFrame::SetVarStruct( dstName, srcMembers, dstFrame ); + BFFStackFrame::SetVarStruct( dstName, varSrc->GetToken(), srcMembers, dstFrame ); } return true; } @@ -1665,7 +1665,7 @@ void BFFParser::CreateBuiltInVariables() { AStackString<> varName( "._WORKING_DIR_" ); ASSERT( BFFStackFrame::GetVarAny( AStackString<>( varName.Get() + 1 ) ) == nullptr ); - BFFStackFrame::SetVarString( varName, FBuild::Get().GetWorkingDir(), &m_BaseStackFrame ); + BFFStackFrame::SetVarString( varName, BFFToken::GetBuiltInToken(), FBuild::Get().GetWorkingDir(), &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } @@ -1673,7 +1673,7 @@ void BFFParser::CreateBuiltInVariables() { AStackString<> varName( "._FASTBUILD_VERSION_STRING_" ); ASSERT( BFFStackFrame::GetVarAny( AStackString<>( varName.Get() + 1 ) ) == nullptr ); - BFFStackFrame::SetVarString( varName, AStackString<>(FBUILD_VERSION_STRING), &m_BaseStackFrame ); + BFFStackFrame::SetVarString( varName, BFFToken::GetBuiltInToken(), AStackString<>(FBUILD_VERSION_STRING), &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } @@ -1681,7 +1681,7 @@ void BFFParser::CreateBuiltInVariables() { AStackString<> varName( "._FASTBUILD_VERSION_" ); ASSERT( BFFStackFrame::GetVarAny( AStackString<>( varName.Get() + 1 ) ) == nullptr ); - BFFStackFrame::SetVarInt( varName, (int32_t)FBUILD_VERSION, &m_BaseStackFrame ); + BFFStackFrame::SetVarInt( varName, BFFToken::GetBuiltInToken(), (int32_t)FBUILD_VERSION, &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } @@ -1691,7 +1691,7 @@ void BFFParser::CreateBuiltInVariables() ASSERT( BFFStackFrame::GetVarAny( AStackString<>( varName.Get() + 1 ) ) == nullptr ); AStackString<> exeName; Env::GetExePath( exeName ); - BFFStackFrame::SetVarString( varName, exeName, &m_BaseStackFrame ); + BFFStackFrame::SetVarString( varName, BFFToken::GetBuiltInToken(), exeName, &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } } @@ -1728,7 +1728,7 @@ void BFFParser::SetBuiltInVariable_CurrentBFFDir( const BFFFile & file ) // Set the variable - always in the base scope const AStackString<> varName( "._CURRENT_BFF_DIR_" ); - BFFStackFrame::SetVarString( varName, relativePath, &m_BaseStackFrame ); + BFFStackFrame::SetVarString( varName, BFFToken::GetBuiltInToken(), relativePath, &m_BaseStackFrame ); // TODO:B Add a mechanism to mark variable as read-only } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h b/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h index 92bf577a4..70c5a73a1 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFParser.h @@ -6,7 +6,6 @@ //------------------------------------------------------------------------------ #include "Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h" #include "Tools/FBuild/FBuildCore/BFF/LinkerNodeFileExistsCache.h" -#include "Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h" #include "Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.h" #include "Core/Env/Assert.h" @@ -17,6 +16,7 @@ // Forward Declarations //------------------------------------------------------------------------------ class BFFFile; +class BFFToken; class BFFTokenRange; class BFFUserFunction; class FileStream; @@ -76,8 +76,8 @@ class BFFParser bool StoreVariableString( const AString & name, const BFFToken * rhsString, const BFFToken * operatorToken, BFFStackFrame * frame ); bool StoreVariableArray( const AString & name, const BFFTokenRange & tokenRange, const BFFToken * operatorIter, BFFStackFrame * frame ); bool StoreVariableStruct( const AString & name, const BFFTokenRange & tokenRange, const BFFToken * operatorToken, BFFStackFrame * frame ); - bool StoreVariableBool( const AString & name, bool value, BFFStackFrame * frame ); - bool StoreVariableInt( const AString & name, int value, BFFStackFrame * frame ); + bool StoreVariableBool( const AString & name, const BFFToken * token, bool value, BFFStackFrame * frame ); + bool StoreVariableInt( const AString & name, const BFFToken * token, int value, BFFStackFrame * frame ); bool StoreVariableToVariable( const AString & dstName, const BFFToken * rhsToken, const BFFToken * operatorToken, BFFStackFrame * dstFrame ); void CreateBuiltInVariables(); diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp index 7e1565a72..539eaf478 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp @@ -5,8 +5,13 @@ //------------------------------------------------------------------------------ #include "BFFStackFrame.h" #include "BFFVariable.h" + +#include "Tools/FBuild/FBuildCore/BFF/Functions/Function.h" +#include "Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h" + #include "Core/Mem/Mem.h" #include "Core/Strings/AStackString.h" +#include "Core/Tracing/Tracing.h" // /*static*/ BFFStackFrame * BFFStackFrame::s_StackHead = nullptr; @@ -62,6 +67,7 @@ void BFFStackFrame::DisconnectStackChain() // SetVarString //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarString( const AString & name, + const BFFToken & token, const AString & value, BFFStackFrame * frame ) { @@ -76,13 +82,14 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, value ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, value ) ); frame->m_Variables.Append( v ); } // SetVarArrayOfStrings //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarArrayOfStrings( const AString & name, + const BFFToken & token, const Array< AString > & values, BFFStackFrame * frame ) { @@ -97,13 +104,16 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, values ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, values ) ); frame->m_Variables.Append( v ); } // SetVarBool //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVarBool( const AString & name, bool value, BFFStackFrame * frame ) +/*static*/ void BFFStackFrame::SetVarBool( const AString & name, + const BFFToken & token, + bool value, + BFFStackFrame * frame ) { frame = frame ? frame : s_StackHead; ASSERT( frame ); @@ -116,13 +126,16 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, value ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, value ) ); frame->m_Variables.Append( v ); } // SetVarInt //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVarInt( const AString & name, int value, BFFStackFrame * frame ) +/*static*/ void BFFStackFrame::SetVarInt( const AString & name, + const BFFToken & token, + int value, + BFFStackFrame * frame ) { frame = frame ? frame : s_StackHead; ASSERT( frame ); @@ -135,13 +148,14 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, value ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, value ) ); frame->m_Variables.Append( v ); } // SetVarStruct //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarStruct( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & members, BFFStackFrame * frame ) { @@ -156,13 +170,14 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, members ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, members ) ); frame->m_Variables.Append( v ); } // SetVarStruct //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarStruct( const AString& name, + const BFFToken & token, Array && members, BFFStackFrame * frame ) { @@ -177,7 +192,7 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable* v = FNEW( BFFVariable( name, Move( members ) ) ); + BFFVariable* v = FNEW( BFFVariable( name, token, Move( members ) ) ); frame->m_Variables.Append( v ); } @@ -185,6 +200,7 @@ void BFFStackFrame::DisconnectStackChain() // SetVarArrayOfStructs //------------------------------------------------------------------------------ /*static*/ void BFFStackFrame::SetVarArrayOfStructs( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & structs, BFFStackFrame * frame ) { @@ -199,21 +215,26 @@ void BFFStackFrame::DisconnectStackChain() } // variable not found at this level, so create it - BFFVariable * v = FNEW( BFFVariable( name, structs, BFFVariable::VAR_ARRAY_OF_STRUCTS ) ); + BFFVariable * v = FNEW( BFFVariable( name, token, structs, BFFVariable::VAR_ARRAY_OF_STRUCTS ) ); frame->m_Variables.Append( v ); } // SetVar //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVar( const BFFVariable * var, BFFStackFrame * frame ) +/*static*/ void BFFStackFrame::SetVar( const BFFVariable * var, + const BFFToken & token, + BFFStackFrame * frame ) { - return SetVar( var, var->GetName(), frame ); + SetVar( var, token, var->GetName(), frame ); } // SetVar //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVar( const BFFVariable * srcVar, const AString & dstName, BFFStackFrame * frame ) +/*static*/ void BFFStackFrame::SetVar( const BFFVariable * srcVar, + const BFFToken & token, + const AString & dstName, + BFFStackFrame * frame ) { frame = frame ? frame : s_StackHead; ASSERT( frame ); @@ -223,12 +244,12 @@ void BFFStackFrame::DisconnectStackChain() switch ( srcVar->GetType() ) { case BFFVariable::VAR_ANY: ASSERT( false ); break; - case BFFVariable::VAR_STRING: SetVarString( dstName, srcVar->GetString(), frame ); break; - case BFFVariable::VAR_BOOL: SetVarBool( dstName, srcVar->GetBool(), frame ); break; - case BFFVariable::VAR_ARRAY_OF_STRINGS: SetVarArrayOfStrings( dstName, srcVar->GetArrayOfStrings(), frame ); break; - case BFFVariable::VAR_INT: SetVarInt( dstName, srcVar->GetInt(), frame ); break; - case BFFVariable::VAR_STRUCT: SetVarStruct( dstName, srcVar->GetStructMembers(), frame ); break; - case BFFVariable::VAR_ARRAY_OF_STRUCTS: SetVarArrayOfStructs( dstName, srcVar->GetArrayOfStructs(), frame ); break; + case BFFVariable::VAR_STRING: SetVarString( dstName, token, srcVar->GetString(), frame ); break; + case BFFVariable::VAR_BOOL: SetVarBool( dstName, token, srcVar->GetBool(), frame ); break; + case BFFVariable::VAR_ARRAY_OF_STRINGS: SetVarArrayOfStrings( dstName, token, srcVar->GetArrayOfStrings(), frame ); break; + case BFFVariable::VAR_INT: SetVarInt( dstName, token, srcVar->GetInt(), frame ); break; + case BFFVariable::VAR_STRUCT: SetVarStruct( dstName, token, srcVar->GetStructMembers(), frame ); break; + case BFFVariable::VAR_ARRAY_OF_STRUCTS: SetVarArrayOfStructs( dstName, token, srcVar->GetArrayOfStructs(), frame ); break; case BFFVariable::MAX_VAR_TYPES: ASSERT( false ); break; } } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h index 5fd8aeafc..73518f98d 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h @@ -10,6 +10,7 @@ // Forward Declarations //------------------------------------------------------------------------------ class AString; +class Function; // BFFStackFrame //------------------------------------------------------------------------------ @@ -23,30 +24,42 @@ class BFFStackFrame // set the value of a variable static void SetVarString( const AString & name, + const BFFToken & token, const AString & value, BFFStackFrame * frame ); static void SetVarArrayOfStrings( const AString & name, + const BFFToken & token, const Array< AString > & values, BFFStackFrame * frame ); static void SetVarBool( const AString & name, + const BFFToken & token, bool value, BFFStackFrame * frame ); static void SetVarInt( const AString & name, + const BFFToken & token, int value, BFFStackFrame * frame ); static void SetVarStruct( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & members, BFFStackFrame * frame ); static void SetVarStruct( const AString & name, + const BFFToken & token, Array && members, BFFStackFrame * frame ); static void SetVarArrayOfStructs( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & structs, BFFStackFrame * frame ); // set from an existing variable - static void SetVar( const BFFVariable * var, BFFStackFrame * frame ); - static void SetVar( const BFFVariable * srcVar, const AString & dstName, BFFStackFrame * frame ); + static void SetVar( const BFFVariable * var, + const BFFToken & token, + BFFStackFrame * frame ); + static void SetVar( const BFFVariable * srcVar, + const BFFToken & token, + const AString & dstName, + BFFStackFrame * frame ); // set from two existing variable static BFFVariable * ConcatVars( const AString & name, diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp index 1f412dda1..b4090e2fb 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp @@ -24,9 +24,10 @@ // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, VarType type ) +BFFVariable::BFFVariable( const AString & name, const BFFToken & token, VarType type ) : m_Name( name ) , m_Type( type ) + , m_Token( token ) { } @@ -35,6 +36,7 @@ BFFVariable::BFFVariable( const AString & name, VarType type ) BFFVariable::BFFVariable( const BFFVariable & other ) : m_Name( other.m_Name ) , m_Type( other.m_Type ) + , m_Token( other.m_Token ) { switch( m_Type ) { @@ -51,46 +53,61 @@ BFFVariable::BFFVariable( const BFFVariable & other ) // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, const AString & value ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + const AString & value ) : m_Name( name ) , m_Type( VAR_STRING ) , m_StringValue( value ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, bool value ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + bool value ) : m_Name( name ) , m_Type( VAR_BOOL ) , m_BoolValue( value ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, const Array< AString > & values ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + const Array< AString > & values ) : m_Name( name ) , m_Type( VAR_ARRAY_OF_STRINGS ) , m_ArrayValues( values ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, int32_t i ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + int32_t i ) : m_Name( name ) , m_Type( VAR_INT ) , m_IntValue( i ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ -BFFVariable::BFFVariable( const AString & name, const Array< const BFFVariable * > & values ) +BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, + const Array< const BFFVariable * > & values ) : m_Name( name ) , m_Type( VAR_STRUCT ) , m_SubVariables( values.GetSize(), true ) + , m_Token( token ) { SetValueStruct( values ); } @@ -98,21 +115,25 @@ BFFVariable::BFFVariable( const AString & name, const Array< const BFFVariable * // CONSTRUCTOR (&&) //------------------------------------------------------------------------------ BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, Array && values ) : m_Name( name ) , m_Type( VAR_STRUCT ) , m_SubVariables( Move( values ) ) + , m_Token( token ) { } // CONSTRUCTOR //------------------------------------------------------------------------------ BFFVariable::BFFVariable( const AString & name, + const BFFToken & token, const Array< const BFFVariable * > & structs, VarType type ) // type for disambiguation : m_Name( name ) , m_Type( VAR_ARRAY_OF_STRUCTS ) , m_SubVariables( structs.GetSize(), true ) + , m_Token( token ) { // type for disambiguation only - sanity check it's the right type ASSERT( type == VAR_ARRAY_OF_STRUCTS ); (void)type; @@ -288,7 +309,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF values.Append( varDst->GetArrayOfStrings() ); } values.Append( varSrc->GetString() ); - BFFVariable *result = FNEW(BFFVariable(dstName, values)); + BFFVariable * result = FNEW( BFFVariable( dstName, m_Token, values ) ); return result; } // Struct to ArrayOfStructs or empty array concatenation @@ -304,7 +325,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF } values.Append( varSrc ); - BFFVariable *result = FNEW( BFFVariable( dstName, values, VAR_ARRAY_OF_STRUCTS ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, m_Token, values, VAR_ARRAY_OF_STRUCTS ) ); return result; } @@ -313,7 +334,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF ( ( ( srcType == BFFVariable::VAR_ARRAY_OF_STRUCTS ) || ( srcType == BFFVariable::VAR_ARRAY_OF_STRINGS ) ) && dstIsEmpty ) ) { const BFFVariable * src = srcIsEmpty ? varDst : varSrc; - BFFVariable * result = FNEW( BFFVariable( dstName, src->m_Type ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, m_Token, src->m_Type ) ); if ( src->m_Type == BFFVariable::VAR_ARRAY_OF_STRINGS ) { result->SetValueArrayOfStrings( src->GetArrayOfStrings() ); @@ -342,7 +363,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF finalValue = varDst->GetString(); finalValue += varSrc->GetString(); - BFFVariable *result = FNEW( BFFVariable( dstName, finalValue ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, varSrc->m_Token, finalValue ) ); return result; } @@ -354,7 +375,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF values.Append( varDst->GetArrayOfStrings() ); values.Append( varSrc->GetArrayOfStrings() ); - BFFVariable *result = FNEW( BFFVariable( dstName, values ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, varSrc->m_Token, values ) ); return result; } @@ -366,7 +387,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF values.Append( varDst->GetArrayOfStructs() ); values.Append( varSrc->GetArrayOfStructs() ); - BFFVariable * result = FNEW(BFFVariable( dstName, values, VAR_ARRAY_OF_STRUCTS ) ); + BFFVariable * result = FNEW(BFFVariable( dstName, varSrc->m_Token, values, VAR_ARRAY_OF_STRUCTS ) ); return result; } @@ -375,7 +396,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF int newVal( varSrc->GetInt() ); newVal += varDst->GetInt(); - BFFVariable * result = FNEW( BFFVariable( dstName, newVal ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, varSrc->m_Token, newVal ) ); return result; } @@ -385,7 +406,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF bool newVal( varSrc->GetBool() ); newVal |= varDst->GetBool(); - BFFVariable * result = FNEW( BFFVariable( dstName, newVal ) ); + BFFVariable * result = FNEW( BFFVariable( dstName, varSrc->m_Token, newVal ) ); return result; } @@ -394,7 +415,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF const Array< const BFFVariable * > & srcMembers = varSrc->GetStructMembers(); const Array< const BFFVariable * > & dstMembers = varDst->GetStructMembers(); - BFFVariable * const result = FNEW( BFFVariable( dstName, BFFVariable::VAR_STRUCT ) ); + BFFVariable * const result = FNEW( BFFVariable( dstName, varSrc->m_Token, BFFVariable::VAR_STRUCT ) ); result->m_SubVariables.SetCapacity( srcMembers.GetSize() + dstMembers.GetSize() ); Array< BFFVariable * > & allMembers = result->m_SubVariables; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.h b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.h index 2848117b8..a5ca1d832 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.h @@ -68,19 +68,21 @@ class BFFVariable static const BFFVariable ** GetMemberByName( const AString & name, const Array< const BFFVariable * > & members ); + const BFFToken & GetToken() const { return m_Token; } + private: friend class BFFStackFrame; explicit BFFVariable( const BFFVariable & other ); - explicit BFFVariable( const AString & name, VarType type ); - explicit BFFVariable( const AString & name, const AString & value ); - explicit BFFVariable( const AString & name, bool value ); - explicit BFFVariable( const AString & name, const Array< AString > & values ); - explicit BFFVariable( const AString & name, int32_t i ); - explicit BFFVariable( const AString & name, const Array< const BFFVariable * > & values ); - explicit BFFVariable( const AString & name, Array && values ); - explicit BFFVariable( const AString & name, const Array< const BFFVariable * > & structs, VarType type ); // type for disambiguation + explicit BFFVariable( const AString & name, const BFFToken & token, VarType type ); + explicit BFFVariable( const AString & name, const BFFToken & token, const AString & value ); + explicit BFFVariable( const AString & name, const BFFToken & token, bool value ); + explicit BFFVariable( const AString & name, const BFFToken & token, const Array< AString > & values ); + explicit BFFVariable( const AString & name, const BFFToken & token, int32_t i ); + explicit BFFVariable( const AString & name, const BFFToken & token, const Array< const BFFVariable * > & values ); + explicit BFFVariable( const AString & name, const BFFToken & token, Array && values ); + explicit BFFVariable( const AString & name, const BFFToken & token, const Array< const BFFVariable * > & structs, VarType type ); // type for disambiguation ~BFFVariable(); BFFVariable & operator =( const BFFVariable & other ) = delete; @@ -104,6 +106,7 @@ class BFFVariable AString m_StringValue; Array< AString > m_ArrayValues; Array< BFFVariable * > m_SubVariables; // Used for struct members of arrays of structs + const BFFToken & m_Token; static const char * s_TypeNames[ MAX_VAR_TYPES ]; }; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionForEach.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionForEach.cpp index f41832146..92d1d88cf 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionForEach.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionForEach.cpp @@ -38,7 +38,7 @@ FunctionForEach::FunctionForEach() //------------------------------------------------------------------------------ /*virtual*/ bool FunctionForEach::ParseFunction( NodeGraph & /*nodeGraph*/, BFFParser & parser, - const BFFToken * /*functionNameStart*/, + const BFFToken * functionNameStart, const BFFTokenRange & headerRange, const BFFTokenRange & bodyRange ) const { @@ -189,11 +189,11 @@ FunctionForEach::FunctionForEach() { if ( arrayVars[ j ]->GetType() == BFFVariable::VAR_ARRAY_OF_STRINGS ) { - BFFStackFrame::SetVarString( localNames[ j ], arrayVars[ j ]->GetArrayOfStrings()[ i ], &loopStackFrame ); + BFFStackFrame::SetVarString( localNames[ j ], *functionNameStart, arrayVars[ j ]->GetArrayOfStrings()[ i ], &loopStackFrame ); } else if ( arrayVars[ j ]->GetType() == BFFVariable::VAR_ARRAY_OF_STRUCTS ) { - BFFStackFrame::SetVarStruct( localNames[ j ], arrayVars[ j ]->GetArrayOfStructs()[ i ]->GetStructMembers(), &loopStackFrame ); + BFFStackFrame::SetVarStruct( localNames[ j ], *functionNameStart, arrayVars[ j ]->GetArrayOfStructs()[ i ]->GetStructMembers(), &loopStackFrame ); } else { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp index f2ec822b8..e703eb173 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp @@ -108,12 +108,12 @@ FunctionUsing::FunctionUsing() sameNameMember = member; continue; } - BFFStackFrame::SetVar( member, frame ); + BFFStackFrame::SetVar( member, *varToken, frame ); } if ( sameNameMember != nullptr ) { - BFFStackFrame::SetVar( sameNameMember, frame ); + BFFStackFrame::SetVar( sameNameMember, *varToken, frame ); } if ( headerIter.IsAtEnd() == false ) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.cpp index 3ac5ab352..8a146ba0b 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.cpp @@ -7,6 +7,15 @@ #include +// Static Data +//------------------------------------------------------------------------------ +/*static*/ const BFFFile BFFToken::s_BuiltInFile( "", AString::GetEmpty() ); +/*static*/ const BFFToken BFFToken::s_BuiltInToken( s_BuiltInFile, + s_BuiltInFile.GetSourceFileContents().Get(), + BFFTokenType::Invalid, + s_BuiltInFile.GetSourceFileContents().Get(), + s_BuiltInFile.GetSourceFileContents().Get() ); + // CONSTRUCTOR //------------------------------------------------------------------------------ BFFToken::BFFToken( const BFFFile & file, diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h index 884c7b9df..19b67b11f 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h @@ -81,6 +81,9 @@ class BFFToken void GetPosInfo( uint32_t & outLine, uint32_t & outColumn, const char * & outLineStart ) const; + // Some variables come from built-in declarations so we need a proxy BFFToken for those + static const BFFToken & GetBuiltInToken() { return s_BuiltInToken; } + private: BFFTokenType m_Type; bool m_Boolean = false; @@ -88,6 +91,10 @@ class BFFToken AString m_String; const BFFFile & m_BFFFile; const char * m_SourcePos = nullptr; + + // Static Data + static const BFFFile s_BuiltInFile; + static const BFFToken s_BuiltInToken; }; //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp b/Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp index 8828bcae3..9b8672149 100644 --- a/Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp +++ b/Code/Tools/FBuild/FBuildCore/Cache/ICache.cpp @@ -16,7 +16,7 @@ AString & outCacheId ) { // cache version - bump if cache format is changed - const char cacheVersion( 'D' ); + const char cacheVersion( 'E' ); // format example: 2377DE32AB045A2D_FED872A1_AB62FEAA23498AAC-32A2B04375A2D7DE.7 outCacheId.Format( "%016" PRIX64 "_%08X_%016" PRIX64 "-%016" PRIX64 ".%c", diff --git a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp index 6bd8317b7..1c379ccfa 100644 --- a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp +++ b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp @@ -331,7 +331,7 @@ bool LightCache::Hash( ObjectNode * node, hashes.Append( file->m_ContentHash ); outIncludes.Append( file->m_FileName ); } - outSourceHash = xxHash::Calc64( hashes.Begin(), hashes.GetSize() * sizeof( uint64_t ) ); + outSourceHash = xxHash3::Calc64( hashes.Begin(), hashes.GetSize() * sizeof( uint64_t ) ); return true; } @@ -365,7 +365,7 @@ void LightCache::Parse( IncludedFile * file, FileStream & f ) // Store hash of file - file->m_ContentHash = xxHash::Calc64( fileContents ); + file->m_ContentHash = xxHash3::Calc64( fileContents ); const char * pos = fileContents.Get(); for (;;) @@ -503,7 +503,7 @@ bool LightCache::ParseDirective_Define( IncludedFile & file, const char * & pos if ( ParseIncludeString( pos, include, includeType ) == false ) { // Not an include. We only care that this exists (not what it resolves to) - file.m_NonIncludeDefines.Append( xxHash::Calc64( macroName ) ); + file.m_NonIncludeDefines.Append( xxHash3::Calc64( macroName ) ); return true; } @@ -649,7 +649,7 @@ void LightCache::ProcessInclude( const AString & include, IncludeType type ) // The macro was not defined as a valid include path // Is it defined at all? - const uint64_t hash = xxHash::Calc64( include ); + const uint64_t hash = xxHash3::Calc64( include ); bool foundNonIncludeDefine = false; for ( const IncludedFile * includedFile : m_AllIncludedFiles ) { @@ -858,7 +858,7 @@ const IncludedFile * LightCache::ProcessIncludeFromIncludePath( const AString & //------------------------------------------------------------------------------ const IncludedFile * LightCache::FileExists( const AString & fileName ) { - const uint64_t fileNameHash = xxHash::Calc64( fileName ); + const uint64_t fileNameHash = xxHash3::Calc64( fileName ); const uint64_t bucketIndex = LIGHTCACHE_HASH_TO_BUCKET( fileNameHash ); IncludedFileBucket & bucket = g_AllIncludedFiles[ bucketIndex ]; // Retrieve from shared cache diff --git a/Code/Tools/FBuild/FBuildCore/FBuildVersion.h b/Code/Tools/FBuild/FBuildCore/FBuildVersion.h index aa48f2e1f..d227ef6a4 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildVersion.h +++ b/Code/Tools/FBuild/FBuildCore/FBuildVersion.h @@ -4,8 +4,8 @@ // Defines //------------------------------------------------------------------------------ -#define FBUILD_VERSION_STRING "v1.09" -#define FBUILD_VERSION (uint32_t)109 +#define FBUILD_VERSION_STRING "v1.10" +#define FBUILD_VERSION (uint32_t)110 #if defined( __WINDOWS__ ) #define FBUILD_VERSION_PLATFORM "Windows" #elif defined( __APPLE__ ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp index 8e8b28d1b..ae63bf32b 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp @@ -190,7 +190,7 @@ CopyDirNode::~CopyDirNode() = default; ASSERT( cn->GetStamp() ); stamps.Append( cn->GetStamp() ); } - m_Stamp = xxHash::Calc64( &stamps[ 0 ], ( stamps.GetSize() * sizeof( uint64_t ) ) ); + m_Stamp = xxHash3::Calc64( &stamps[ 0 ], ( stamps.GetSize() * sizeof( uint64_t ) ) ); } return NODE_RESULT_OK; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp index f1534e216..783ac27b8 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp @@ -234,7 +234,7 @@ DirectoryListNode::~DirectoryListNode() = default; ms.Write( file.IsReadOnly() ); } } - m_Stamp = xxHash::Calc64( ms.GetData(), ms.GetSize() ); + m_Stamp = xxHash3::Calc64( ms.GetData(), ms.GetSize() ); } return NODE_RESULT_OK; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index 0f090e27a..fe46dcf36 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -306,7 +306,7 @@ NodeGraph::LoadResult NodeGraph::Load( ConstMemoryStream & stream, const char * return LoadResult::LOAD_ERROR; // error reading } - const uint64_t dataHash = xxHash::Calc64( mem.Get(), size ); + const uint64_t dataHash = xxHash3::Calc64( mem.Get(), size ); if ( dataHash == usedFiles[ i ].m_DataHash ) { // file didn't change, update stored timestamp to save time on the next run @@ -537,7 +537,7 @@ void NodeGraph::Save( MemoryStream & stream, const char* nodeGraphDBFile ) const char * data = static_cast( stream.GetDataMutable() ); const char * content = ( data + sizeof(NodeGraphHeader) ); const size_t remainingSize = ( stream.GetSize() - sizeof(NodeGraphHeader) ); - const uint64_t hash = xxHash::Calc64( content, remainingSize ); + const uint64_t hash = xxHash3::Calc64( content, remainingSize ); // Update hash in header NodeGraphHeader * headerToUpdate = reinterpret_cast( data ); @@ -1862,7 +1862,7 @@ bool NodeGraph::ReadHeaderAndUsedFiles( ConstMemoryStream & nodeGraphStream, con ASSERT( tell == sizeof( NodeGraphHeader ) ); // Stream should be after header const char* data = ( static_cast( nodeGraphStream.GetData() ) + tell ); const size_t remainingSize = ( nodeGraphStream.GetSize() - tell ); - const uint64_t hash = xxHash::Calc64( data, remainingSize ); + const uint64_t hash = xxHash3::Calc64( data, remainingSize ); if ( hash != ngh.GetContentHash() ) { return false; // DB is corrupt diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index f4a3c980a..59880707d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -63,7 +63,7 @@ class NodeGraphHeader } inline ~NodeGraphHeader() = default; - enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 168 }; + enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 170 }; bool IsValid() const; bool IsCompatibleVersion() const { return m_Version == NODE_GRAPH_CURRENT_VERSION; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index 1ae95d74a..e5649bcf5 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -233,6 +233,7 @@ ObjectListNode::ObjectListNode() if ( m_CompilerOutputPath.IsEmpty() ) { Error::Error_1101_MissingProperty( iter, function, AStackString<>( "CompilerOutputPath" ) ); + return false; } } @@ -467,7 +468,7 @@ ObjectListNode::~ObjectListNode() = default; } // create the object that will compile the above file - if ( CreateDynamicObjectNode( nodeGraph, n->GetName(), AString::GetEmpty() ) == false ) + if ( CreateDynamicObjectNode( nodeGraph, n->GetName(), objListNode->GetCompilerOutputPath() ) == false ) { return false; // CreateDynamicObjectNode will have emitted error } @@ -574,7 +575,7 @@ ObjectListNode::~ObjectListNode() = default; ASSERT( on->GetStamp() ); stamps.Append( on->GetStamp() ); } - m_Stamp = xxHash::Calc64( &stamps[0], ( stamps.GetSize() * sizeof(uint64_t) ) ); + m_Stamp = xxHash3::Calc64( &stamps[0], ( stamps.GetSize() * sizeof(uint64_t) ) ); } return NODE_RESULT_OK; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h index 992fb7fb9..afce005ea 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h @@ -38,6 +38,7 @@ class ObjectListNode : public Node void GetInputFiles( Args & fullArgs, const AString & pre, const AString & post, bool objectsInsteadOfLibs ) const; void GetInputFiles( Array< AString > & files ) const; + inline const AString & GetCompilerOutputPath() const { return m_CompilerOutputPath; } inline const AString & GetCompilerOptions() const { return m_CompilerOptions; } inline const AString & GetCompiler() const { return m_Compiler; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp index cf32de103..0b139f1df 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp @@ -1334,7 +1334,7 @@ const AString & ObjectNode::GetCacheName( Job * job ) const // hash the pre-processed input data ASSERT( m_LightCacheKey || job->GetData() ); - const uint64_t preprocessedSourceKey = m_LightCacheKey ? m_LightCacheKey : xxHash::Calc64( job->GetData(), job->GetDataSize() ); + const uint64_t preprocessedSourceKey = m_LightCacheKey ? m_LightCacheKey : xxHash3::Calc64( job->GetData(), job->GetDataSize() ); ASSERT( preprocessedSourceKey ); // hash the build "environment" @@ -1408,7 +1408,7 @@ bool ObjectNode::RetrieveFromCache( Job * job ) uint64_t pchKey = 0; if ( IsCreatingPCH() && IsMSVC() ) { - pchKey = xxHash::Calc64( cacheData, cacheDataSize ); + pchKey = xxHash3::Calc64( cacheData, cacheDataSize ); } const uint32_t startDecompress = uint32_t( t.GetElapsedMS() ); @@ -1590,7 +1590,7 @@ void ObjectNode::WriteToCache_FromCompressedData( Job * job, // Dependent objects need to know the PCH key to be able to pull from the cache if ( IsCreatingPCH() && IsMSVC() ) { - m_PCHCacheKey = xxHash::Calc64( compressedData, compressedDataSize ); + m_PCHCacheKey = xxHash3::Calc64( compressedData, compressedDataSize ); } const uint32_t cachingTime = uint32_t( t.GetElapsedMS() ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp index 9755575a6..b6a85ba5d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp @@ -299,7 +299,7 @@ SLNNode::~SLNNode() = default; } // Record stamp representing the contents of the files - m_Stamp = xxHash::Calc64( sln ); + m_Stamp = xxHash3::Calc64( sln ); return NODE_RESULT_OK; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp index f9c929b34..fc52a1ec4 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp @@ -514,7 +514,7 @@ UnityNode::~UnityNode() m_UnityFileNames.Append( unityName ); } - stamps.Append( xxHash::Calc64( output.Get(), output.GetLength() ) ); + stamps.Append( xxHash3::Calc64( output.Get(), output.GetLength() ) ); // need to write the unity file? bool needToWrite = false; @@ -584,7 +584,7 @@ UnityNode::~UnityNode() // Calculate final hash to represent generation of Unity files ASSERT( stamps.GetSize() == m_NumUnityFilesToCreate ); - m_Stamp = xxHash::Calc64( &stamps[ 0 ], stamps.GetSize() * sizeof( uint64_t ) ); + m_Stamp = xxHash3::Calc64( &stamps[ 0 ], stamps.GetSize() * sizeof( uint64_t ) ); // Track "nounity" status in the lest significant bit if (noUnity) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp index b6bfef1b5..1c8afe2d4 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp @@ -311,7 +311,7 @@ VCXProjectNode::~VCXProjectNode() = default; } // Record stamp representing the contents of the files - m_Stamp = xxHash::Calc64( project ) + xxHash::Calc64( filters ); + m_Stamp = xxHash3::Calc64( project ) + xxHash3::Calc64( filters ); return NODE_RESULT_OK; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp index 970dc3222..4d9dbf973 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp @@ -251,7 +251,7 @@ XCodeProjectNode::~XCodeProjectNode() = default; } // Combine hash - stamp += xxHash::Calc64( output ); + stamp += xxHash3::Calc64( output ); } // Get folder containing project.pbxproj @@ -285,7 +285,7 @@ XCodeProjectNode::~XCodeProjectNode() = default; } // Combine hash - stamp += xxHash::Calc64( output ); + stamp += xxHash3::Calc64( output ); } // Generate .xcscheme file @@ -306,7 +306,7 @@ XCodeProjectNode::~XCodeProjectNode() = default; } // Combine hash - stamp += xxHash::Calc64( output ); + stamp += xxHash3::Calc64( output ); } // Record stamp representing the contents of the files diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp index ddc7c3f24..49da73971 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp @@ -37,10 +37,8 @@ BuildProfiler::~BuildProfiler() = default; void BuildProfiler::StartMetricsGathering() { PROFILE_FUNCTION; - - ASSERT( m_Thread == INVALID_THREAD_HANDLE ); m_ThreadExit.Store( false ); - m_Thread = Thread::CreateThread( MetricsThreadWrapper, "BuildProfileMetrics" ); + m_Thread.Start( MetricsThreadWrapper, "BuildProfileMetrics" ); } // StopMetricsGathering @@ -48,13 +46,9 @@ void BuildProfiler::StartMetricsGathering() void BuildProfiler::StopMetricsGathering() { PROFILE_FUNCTION; - - ASSERT( m_Thread != INVALID_THREAD_HANDLE ); m_ThreadExit.Store( true ); m_ThreadSignalSemaphore.Signal(); - Thread::WaitForThread( m_Thread ); - Thread::CloseHandle( m_Thread ); - m_Thread = INVALID_THREAD_HANDLE; + m_Thread.Join(); } // RecordLocal @@ -113,7 +107,7 @@ void BuildProfiler::RecordRemote( uint32_t workerId, bool BuildProfiler::SaveJSON( const FBuildOptions & options, const char * fileName ) { // Thread must be stopped - ASSERT( m_Thread == INVALID_THREAD_HANDLE ); + ASSERT( m_Thread.IsRunning() == false ); // Record time taken to save (can't use regular macros since we're process those) const int64_t saveStart = Timer::GetNow(); diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.h b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.h index 35823cf4a..f2a460430 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.h +++ b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.h @@ -103,7 +103,7 @@ class BuildProfiler : public Singleton Mutex m_Mutex; Atomic m_ThreadExit{ false }; Semaphore m_ThreadSignalSemaphore; - Thread::ThreadHandle m_Thread = INVALID_THREAD_HANDLE; + Thread m_Thread; Array m_Events; Array m_Metrics; Array m_WorkerInfo; diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp index b7340bfc7..f0ee99a2a 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp @@ -216,7 +216,7 @@ bool ToolManifest::DoBuild( const Dependencies & dependencies ) *pos = xxHash::Calc32( relativePath ); ++pos; } - m_ToolId = xxHash::Calc64( mem, memSize ); + m_ToolId = xxHash3::Calc64( mem, memSize ); FREE( mem ); // update time stamp (most recent file in manifest) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp index fae36c505..8a5a3e971 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp @@ -46,11 +46,7 @@ Client::Client( const Array< AString > & workerList, // allocate space for server states m_ServerList.SetSize( workerList.GetSize() ); - m_Thread = Thread::CreateThread( ThreadFuncStatic, - "Client", - ( 64 * KILOBYTE ), - this ); - ASSERT( m_Thread ); + m_Thread.Start( ThreadFuncStatic, "Client", this ); } // DESTRUCTOR @@ -62,11 +58,9 @@ Client::~Client() SetShuttingDown(); m_ShouldExit.Store( true ); - Thread::WaitForThread( m_Thread ); + m_Thread.Join(); ShutdownAllConnections(); - - Thread::CloseHandle( m_Thread ); } //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.h b/Code/Tools/FBuild/FBuildCore/Protocol/Client.h index 6a12213f9..8b3462475 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.h @@ -69,7 +69,7 @@ class Client : public TCPConnectionPool Array< AString > m_WorkerList; // workers to connect to Atomic m_ShouldExit; // signal from main thread bool m_DetailedLogging; - Thread::ThreadHandle m_Thread; // the thread to find and manage workers + Thread m_Thread; // the thread to find and manage workers // state Timer m_StatusUpdateTimer; diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index acf89fc38..cf5f0db68 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -34,11 +34,7 @@ Server::Server( uint32_t numThreadsInJobQueue ) { m_JobQueueRemote = FNEW( JobQueueRemote( numThreadsInJobQueue ? numThreadsInJobQueue : Env::GetNumProcessors() ) ); - m_Thread = Thread::CreateThread( ThreadFuncStatic, - "Server", - ( 64 * KILOBYTE ), - this ); - ASSERT( m_Thread ); + m_Thread.Start( ThreadFuncStatic, "Server", this ); } // DESTRUCTOR @@ -47,12 +43,10 @@ Server::~Server() { m_ShouldExit.Store( true ); JobQueueRemote::Get().WakeMainThread(); - Thread::WaitForThread( m_Thread ); + m_Thread.Join(); ShutdownAllConnections(); - Thread::CloseHandle( m_Thread ); - FDELETE m_JobQueueRemote; for ( ToolManifest * tool : m_Tools ) @@ -296,7 +290,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgConn // take note of initial status of client ClientState * cs = (ClientState *)connection->GetUserData(); MutexHolder mh( cs->m_Mutex ); - cs->m_NumJobsAvailable = msg->GetNumJobsAvailable(); + cs->m_NumJobsAvailable.Store( msg->GetNumJobsAvailable() ); cs->m_ProtocolVersionMinor = msg->GetProtocolVersionMinor(); cs->m_HostName = msg->GetHostName(); } @@ -307,8 +301,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgStat { // take note of latest status of client ClientState * cs = (ClientState *)connection->GetUserData(); - MutexHolder mh( cs->m_Mutex ); - cs->m_NumJobsAvailable = msg->GetNumJobsAvailable(); + cs->m_NumJobsAvailable.Store( msg->GetNumJobsAvailable() ); // Wake main thread to request jobs JobQueueRemote::Get().WakeMainThread(); @@ -320,9 +313,8 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgNoJo { // We requested a job, but the client didn't have any left ClientState * cs = (ClientState *)connection->GetUserData(); - MutexHolder mh( cs->m_Mutex ); - ASSERT( cs->m_NumJobsRequested > 0 ); - cs->m_NumJobsRequested--; + ASSERT( cs->m_NumJobsRequested.Load() > 0 ); + cs->m_NumJobsRequested.Decrement(); } // Process( MsgJob ) @@ -331,11 +323,12 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgJob { ClientState * cs = (ClientState *)connection->GetUserData(); { + ASSERT( cs->m_NumJobsRequested.Load() > 0 ); + cs->m_NumJobsRequested.Decrement(); + cs->m_NumJobsActive.Increment(); + MutexHolder mh( cs->m_Mutex ); - ASSERT( cs->m_NumJobsRequested > 0 ); - cs->m_NumJobsRequested--; - cs->m_NumJobsActive++; - + // deserialize job ConstMemoryStream ms( payload, payloadSize ); @@ -620,24 +613,26 @@ void Server::FindNeedyClients() } PROFILE_FUNCTION; + + // determine job availability + int32_t availableJobs = (int32_t)WorkerThreadRemote::GetNumCPUsToUse(); + if ( availableJobs == 0 ) + { + return; + } + ++availableJobs; // over request to parallelize building/network transfers + { MutexHolder mh( m_ClientListMutex ); - // determine job availability - int availableJobs = (int)WorkerThreadRemote::GetNumCPUsToUse(); - if ( availableJobs == 0 ) + // determine if all available job slots are in use + for ( const ClientState * cs : m_ClientList ) { - return; - } - ++availableJobs; // over request to parallelize building/network transfers - - for ( ClientState * cs : m_ClientList ) - { - MutexHolder mh2( cs->m_Mutex ); - // any jobs requested or in progress reduce the available count - const int32_t reservedJobs = (int32_t)( cs->m_NumJobsRequested + cs->m_NumJobsActive ); + const uint32_t jobsRequested = cs->m_NumJobsRequested.Load(); + const uint32_t jobsActive = cs->m_NumJobsActive.Load(); + const int32_t reservedJobs = static_cast( jobsRequested + jobsActive ); availableJobs -= reservedJobs; if ( availableJobs <= 0 ) { @@ -658,20 +653,32 @@ void Server::FindNeedyClients() for ( ClientState * cs : m_ClientList ) { - MutexHolder mh2( cs->m_Mutex ); + const uint32_t reservedJobs = cs->m_NumJobsRequested.Load(); - const size_t reservedJobs = cs->m_NumJobsRequested; - - if ( reservedJobs >= cs->m_NumJobsAvailable ) + if ( reservedJobs >= cs->m_NumJobsAvailable.Load() ) { continue; // we've maxed out the requests to this worker } // request job from this client - msg.Send( cs->m_Connection ); - cs->m_NumJobsRequested++; + { + // Acquire the lock but don't wait if unavailable + TryMutexHolder tryLock( cs->m_Mutex ); + if ( tryLock.IsLocked() == false ) + { + continue; // Skip this worker for now + } + cs->m_NumJobsRequested.Increment(); // Must be before Send() to ensure consistent counts + msg.Send( cs->m_Connection ); + } availableJobs--; anyJobsRequested = true; + + // Have we consumed all of our requests? + if ( availableJobs == 0 ) + { + break; + } } // if we did a pass and couldn't request any more jobs, then bail out @@ -718,9 +725,10 @@ void Server::FinalizeCompletedJobs() ms.WriteBuffer( job->GetData(), job->GetDataSize() ); { + ASSERT( cs->m_NumJobsActive.Load() > 0 ); + cs->m_NumJobsActive.Decrement(); + MutexHolder mh2( cs->m_Mutex ); - ASSERT( cs->m_NumJobsActive ); - cs->m_NumJobsActive--; if ( job->GetResultCompressionLevel() == 0 ) { diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h index 2b731e421..8be313ed8 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h @@ -67,15 +67,15 @@ class Server : public TCPConnectionPool , m_WaitingJobs( 16, true ) {} - inline bool operator < ( const ClientState & other ) const { return ( m_NumJobsAvailable > other.m_NumJobsAvailable ); } + inline bool operator < ( const ClientState & other ) const { return ( m_NumJobsAvailable.Load() > other.m_NumJobsAvailable.Load() ); } Mutex m_Mutex; const Protocol::IMessage * m_CurrentMessage = nullptr; const ConnectionInfo * m_Connection = nullptr; - uint32_t m_NumJobsAvailable = 0; - uint32_t m_NumJobsRequested = 0; - uint32_t m_NumJobsActive = 0; + Atomic m_NumJobsAvailable; + Atomic m_NumJobsRequested; + Atomic m_NumJobsActive; uint8_t m_ProtocolVersionMinor = 0; AString m_HostName; @@ -88,7 +88,7 @@ class Server : public TCPConnectionPool JobQueueRemote * m_JobQueueRemote; Atomic m_ShouldExit; // signal from main thread - Thread::ThreadHandle m_Thread; // the thread to manage workload + Thread m_Thread; // the thread to manage workload Mutex m_ClientListMutex; Array< ClientState * > m_ClientList; diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp index 84799dc3e..95bea6cf3 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp @@ -41,19 +41,14 @@ void WorkerThread::Init() PROFILE_FUNCTION; // Start thread - Thread::ThreadHandle h = Thread::CreateThread( ThreadWrapperFunc, - "WorkerThread", - MEGABYTE, - this ); - ASSERT( h != nullptr ); - Thread::DetachThread( h ); - Thread::CloseHandle( h ); // we don't want to keep this, so free it now + m_Thread.Start( ThreadWrapperFunc, "WorkerThread", this, MEGABYTE ); } //------------------------------------------------------------------------------ WorkerThread::~WorkerThread() { ASSERT( m_Exited.Load() ); + m_Thread.Join(); } // InitTmpDir diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h index da05980a5..61ef5b3ad 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h @@ -8,6 +8,7 @@ #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" #include "Core/Process/Semaphore.h" +#include "Core/Process/Thread.h" #include "Core/Strings/AStackString.h" #include "Core/Strings/AString.h" @@ -49,6 +50,7 @@ class WorkerThread virtual void Main(); // signal to exit thread + Thread m_Thread; Atomic m_ShouldExit; Atomic m_Exited; uint16_t m_ThreadIndex; diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/ObjectListChaining/A/file.1.cpp b/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/ObjectListChaining/A/file.1.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/Code/Tools/FBuild/FBuildTest/FBuildTest.bff b/Code/Tools/FBuild/FBuildTest/FBuildTest.bff index 83edb1493..509e41c1a 100644 --- a/Code/Tools/FBuild/FBuildTest/FBuildTest.bff +++ b/Code/Tools/FBuild/FBuildTest/FBuildTest.bff @@ -54,6 +54,7 @@ 'TestFrameWork-Lib-$Platform$-$BuildConfigName$', 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/$ProjectName$$ExeExtension$' #if __WINDOWS__ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp index a013272da..be1f266e2 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp @@ -426,14 +426,14 @@ void TestDistributed::ShutdownMemoryLeak() const } }; bool detectedDistributedJobs = false; - Thread::ThreadHandle h = Thread::CreateThread( Helper::AbortBuild, nullptr, 64 * KILOBYTE, &detectedDistributedJobs ); + Thread thread; + thread.Start( Helper::AbortBuild,"TestDistributed", &detectedDistributedJobs ); // Start build and check it was aborted TEST_ASSERT( fBuild.Build( "ShutdownMemoryLeak" ) == false ); TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Error: BUILD FAILED: ShutdownMemoryLeak" ) ); - Thread::WaitForThread( h ); - Thread::CloseHandle( h ); + thread.Join(); TEST_ASSERT( detectedDistributedJobs ); } diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp index 4ed69ab8f..3b7936ec7 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp @@ -84,14 +84,14 @@ void TestFastCancel::Cancel() const SystemMutex * mutexes[] = { &mutex1, &mutex2, &mutex3, &mutex4 }; // Create thread that will abort build once all processes are spawned - Thread::ThreadHandle h = Thread::CreateThread( CancelHelperThread ); + Thread thread; + thread.Start( CancelHelperThread ); // Start build and check it was aborted TEST_ASSERT( fBuild.Build( "Cancel" ) == false ); TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Error: BUILD FAILED: Cancel" ) ); - Thread::WaitForThread( h ); - Thread::CloseHandle( h ); + thread.Join(); // Ensure that processes were killed for ( SystemMutex * mutex : mutexes ) diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp index 3b42387b0..8fba02ce2 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp @@ -859,7 +859,7 @@ void TestGraph::DBVersionChanged() const { // Generate a fake old version headers NodeGraphHeader header; - header.SetContentHash( xxHash::Calc64( "", 0 ) ); + header.SetContentHash( xxHash3::Calc64( "", 0 ) ); MemoryStream ms; ms.WriteBuffer( &header, sizeof( header ) ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp index 5f1e4a348..ce7c0277e 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp @@ -285,7 +285,7 @@ void TestNodeReflection::String_Optional_Set() const TestHelper helper( new Node_String_Optional ); // Set string - helper.m_Frame.SetVarString( AStackString<>( ".String" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".String" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check string was set TEST_ASSERT( helper.Populate() ); @@ -299,7 +299,7 @@ void TestNodeReflection::String_Optional_Empty() const TestHelper helper( new Node_String_Optional ); // Set string to empty - helper.m_Frame.SetVarString( AStackString<>( ".String" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".String" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Check ok (setting empty on optional property is ok) TEST_ASSERT( helper.Populate() ); @@ -333,7 +333,7 @@ void TestNodeReflection::String_Required_Set() const TestHelper helper( new Node_String_Required ); // Set string - helper.m_Frame.SetVarString( AStackString<>( ".String" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".String" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check string was set TEST_ASSERT( helper.Populate() ); @@ -347,7 +347,7 @@ void TestNodeReflection::String_Required_Empty() const TestHelper helper( new Node_String_Required ); // Set string to empty - helper.m_Frame.SetVarString( AStackString<>( ".String" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".String" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Check for failure (required strings can't be empty) TEST_ASSERT( helper.Populate() == false ); @@ -384,7 +384,7 @@ void TestNodeReflection::ArrayOfStrings_Optional_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check array was set TEST_ASSERT( helper.Populate() ); @@ -400,7 +400,7 @@ void TestNodeReflection::ArrayOfStrings_Optional_Empty() const // Set array to empty Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Check ok (setting empty on optional property is ok) TEST_ASSERT( helper.Populate() ); @@ -417,7 +417,7 @@ void TestNodeReflection::ArrayOfStrings_Optional_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -453,7 +453,7 @@ void TestNodeReflection::ArrayOfStrings_Required_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check array was set TEST_ASSERT( helper.Populate() ); @@ -469,7 +469,7 @@ void TestNodeReflection::ArrayOfStrings_Required_Empty() const // Set array to empty Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Check for failure (can't set empty array if property is required) TEST_ASSERT( helper.Populate() == false ); @@ -486,7 +486,7 @@ void TestNodeReflection::ArrayOfStrings_Required_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".ArrayOfStrings" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -519,7 +519,7 @@ void TestNodeReflection::MetaFile_String_Optional_Set() const TestHelper helper( new Node_MetaFile_String_Optional ); // Push a string - helper.m_Frame.SetVarString( AStackString<>( ".File" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".File" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -534,7 +534,7 @@ void TestNodeReflection::MetaFile_String_Optional_Empty() const TestHelper helper( new Node_MetaFile_String_Optional ); // Push an empty string - helper.m_Frame.SetVarString( AStackString<>( ".File" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".File" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Ok for property to be empty because it is optional TEST_ASSERT( helper.Populate() == true ); @@ -567,7 +567,7 @@ void TestNodeReflection::MetaFile_String_Required_Set() const TestHelper helper( new Node_MetaFile_String_Required ); // Push a string - helper.m_Frame.SetVarString( AStackString<>( ".File" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".File" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -582,7 +582,7 @@ void TestNodeReflection::MetaFile_String_Required_Empty() const TestHelper helper( new Node_MetaFile_String_Required ); // Push an empty string - helper.m_Frame.SetVarString( AStackString<>( ".File" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".File" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Check that populating properties fails and that appropriate error is reported TEST_ASSERT( helper.Populate() == false ); @@ -617,7 +617,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Optional_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check the property was set and converted to a full paths TEST_ASSERT( helper.Populate() == true ); @@ -634,7 +634,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Optional_Empty() const // Set string array with an empty element in in Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Ok for property to be empty because it is optional TEST_ASSERT( helper.Populate() == true ); @@ -650,7 +650,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Optional_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -686,7 +686,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Required_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -703,7 +703,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Required_Empty() const // Set string array with an empty element in in Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Check that populating properties fails and that appropriate error is reported TEST_ASSERT( helper.Populate() == false ); @@ -720,7 +720,7 @@ void TestNodeReflection::MetaFile_ArrayOfStrings_Required_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Files" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -753,7 +753,7 @@ void TestNodeReflection::MetaPath_String_Optional_Set() const TestHelper helper( new Node_MetaPath_String_Optional ); // Push a string - helper.m_Frame.SetVarString( AStackString<>( ".Path" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".Path" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -768,7 +768,7 @@ void TestNodeReflection::MetaPath_String_Optional_Empty() const TestHelper helper( new Node_MetaPath_String_Optional ); // Push an empty string - helper.m_Frame.SetVarString( AStackString<>( ".Path" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".Path" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Ok for property to be empty because it is optional TEST_ASSERT( helper.Populate() == true ); @@ -801,7 +801,7 @@ void TestNodeReflection::MetaPath_String_Required_Set() const TestHelper helper( new Node_MetaPath_String_Required ); // Push a string - helper.m_Frame.SetVarString( AStackString<>( ".Path" ), AStackString<>( "value" ), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".Path" ), BFFToken::GetBuiltInToken(), AStackString<>( "value" ), nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -816,7 +816,7 @@ void TestNodeReflection::MetaPath_String_Required_Empty() const TestHelper helper( new Node_MetaPath_String_Required ); // Push an empty string - helper.m_Frame.SetVarString( AStackString<>( ".Path" ), AString::GetEmpty(), nullptr ); + helper.m_Frame.SetVarString( AStackString<>( ".Path" ), BFFToken::GetBuiltInToken(), AString::GetEmpty(), nullptr ); // Check that populating properties fails and that appropriate error is reported TEST_ASSERT( helper.Populate() == false ); @@ -851,7 +851,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Optional_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check the property was set and converted to a full paths TEST_ASSERT( helper.Populate() == true ); @@ -868,7 +868,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Optional_Empty() const // Set string array with an empty element in in Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Ok for property to be empty because it is optional TEST_ASSERT( helper.Populate() == true ); @@ -884,7 +884,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Optional_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); @@ -920,7 +920,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Required_Set() const // Set string array Array strings; strings.EmplaceBack( "value" ); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check the property was set and converted to a full path TEST_ASSERT( helper.Populate() == true ); @@ -937,7 +937,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Required_Empty() const // Set string array with an empty element in in Array empty; - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), empty, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), empty, nullptr ); // Check that populating properties fails and that appropriate error is reported TEST_ASSERT( helper.Populate() == false ); @@ -954,7 +954,7 @@ void TestNodeReflection::MetaPath_ArrayOfStrings_Required_EmptyElement() const Array strings; strings.EmplaceBack( "value" ); strings.EmplaceBack(); - helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), strings, nullptr ); + helper.m_Frame.SetVarArrayOfStrings( AStackString<>( ".Paths" ), BFFToken::GetBuiltInToken(), strings, nullptr ); // Check failure (empty strings in arrays are not allowed) TEST_ASSERT( helper.Populate() == false ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp index ad211b3ae..ef9760769 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp @@ -183,7 +183,7 @@ void TestObjectList::ObjectListChaining() const // Check stats // Seen, Built, Type CheckStatsNode( 2, 2, Node::OBJECT_LIST_NODE ); - CheckStatsNode( 2, 2, Node::OBJECT_NODE ); + CheckStatsNode( 4, 4, Node::OBJECT_NODE ); CheckStatsNode( 1, 1, Node::DIRECTORY_LIST_NODE ); fBuild.SerializeDepGraphToText( "ObjectList2", depGraphText1 ); @@ -198,7 +198,7 @@ void TestObjectList::ObjectListChaining() const // Check stats // Seen, Built, Type CheckStatsNode( 2, 0, Node::OBJECT_LIST_NODE ); - CheckStatsNode( 2, 0, Node::OBJECT_NODE ); + CheckStatsNode( 4, 0, Node::OBJECT_NODE ); CheckStatsNode( 1, 1, Node::DIRECTORY_LIST_NODE ); } @@ -213,7 +213,7 @@ void TestObjectList::ObjectListChaining() const // Check stats // Seen, Built, Type CheckStatsNode( 2, 0, Node::OBJECT_LIST_NODE ); - CheckStatsNode( 2, 0, Node::OBJECT_NODE ); + CheckStatsNode( 4, 0, Node::OBJECT_NODE ); CheckStatsNode( 1, 1, Node::DIRECTORY_LIST_NODE ); fBuild.SerializeDepGraphToText( "ObjectList2", depGraphText2 ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestVariableStack.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestVariableStack.cpp index 31825ec5d..d87ea18e7 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestVariableStack.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestVariableStack.cpp @@ -7,6 +7,7 @@ #include "Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h" #include "Tools/FBuild/FBuildCore/BFF/BFFVariable.h" +#include "Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFToken.h" #include "Core/Strings/AStackString.h" @@ -46,7 +47,7 @@ void TestVariableStack::TestStackFramesAdditional() const { // a stack frame with a variable BFFStackFrame sf1; - BFFStackFrame::SetVarString( AStackString<>( "myVarA" ), AStackString<>( "valueA" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVarA" ), BFFToken::GetBuiltInToken(), AStackString<>( "valueA" ), nullptr ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarA" ) ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarA" )->GetString() == "valueA" ); @@ -54,7 +55,7 @@ void TestVariableStack::TestStackFramesAdditional() const // another stack frame { BFFStackFrame sf2; - BFFStackFrame::SetVarString( AStackString<>( "myVarB" ), AStackString<>( "valueB" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVarB" ), BFFToken::GetBuiltInToken(), AStackString<>( "valueB" ), nullptr ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarA" ) ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarA" )->GetString() == "valueA" ); TEST_ASSERT( BFFStackFrame::GetVar( "myVarB" ) ); @@ -73,7 +74,7 @@ void TestVariableStack::TestStackFramesOverride() const { // a stack frame with a variable BFFStackFrame sf1; - BFFStackFrame::SetVarString( AStackString<>( "myVar" ), AStackString<>( "originalValue" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVar" ), BFFToken::GetBuiltInToken(), AStackString<>( "originalValue" ), nullptr ); TEST_ASSERT( BFFStackFrame::GetVar( "myVar" ) ); TEST_ASSERT( BFFStackFrame::GetVar( "myVar" )->GetString() == "originalValue" ); @@ -82,7 +83,7 @@ void TestVariableStack::TestStackFramesOverride() const { // which replaces the same variable BFFStackFrame sf2; - BFFStackFrame::SetVarString( AStackString<>( "myVar" ), AStackString<>( "replacedValue" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVar" ), BFFToken::GetBuiltInToken(), AStackString<>( "replacedValue" ), nullptr ); // we should get the replaced value TEST_ASSERT( BFFStackFrame::GetVar( "myVar" ) ); @@ -102,7 +103,7 @@ void TestVariableStack::TestStackFramesParent() const // a stack frame with a variable BFFStackFrame sf1; - BFFStackFrame::SetVarString( AStackString<>( "myVar" ), AStackString<>( "originalValue" ), nullptr ); + BFFStackFrame::SetVarString( AStackString<>( "myVar" ), BFFToken::GetBuiltInToken(), AStackString<>( "originalValue" ), nullptr ); TEST_ASSERT( BFFStackFrame::GetVar( "myVar", &sf1 ) ); TEST_ASSERT( BFFStackFrame::GetVar( "myVar", &sf1 )->GetString() == "originalValue" ); @@ -118,7 +119,7 @@ void TestVariableStack::TestStackFramesParent() const TEST_ASSERT( BFFStackFrame::GetVar( "myVar", &sf2 ) == nullptr ); // which replaces the same variable - BFFStackFrame::SetVarString( AStackString<>( "myVar" ), AStackString<>( "replacedValue" ), &sf2 ); + BFFStackFrame::SetVarString( AStackString<>( "myVar" ), BFFToken::GetBuiltInToken(), AStackString<>( "replacedValue" ), &sf2 ); // we should bet the original value TEST_ASSERT( BFFStackFrame::GetVar( "myVar", &sf1 ) ); diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff index d995390a1..d7e37a276 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff @@ -79,6 +79,7 @@ 'OSUI-Lib-$Platform$-$BuildConfigName$' 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' + 'xxHash-Lib-$Platform$-$BuildConfigName$' } #if __WINDOWS__ + 'FBuildWorker-Res-$Platform$-$BuildConfigName$' @@ -134,7 +135,20 @@ ^ProjectConfigs + .ProjectConfig #endif } - + + // Create Universal binaries + #if CLANG_SUPPORTS_ARMOSX + ForEach( .BuildConfig in .BuildConfigs ) + { + Using( .BuildConfig ) + If( .Platform == 'ARMOSX' ) + { + CreateUniversalBinary( .ProjectName, .ProjectPath, .BuildConfigName, .OutputBase ) + ^'Targets_OSX_$BuildConfigName$' + { '$ProjectName$-OSX-$BuildConfigName$' } + } + } + #endif + // Aliases //-------------------------------------------------------------------------- CreateCommonAliases( .ProjectName ) diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp index 82f400ee5..3eeb3ccba 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp @@ -31,7 +31,8 @@ FBuildWorkerOptions::FBuildWorkerOptions() : m_OverrideWorkMode( false ), m_WorkMode( WorkerSettings::WHEN_IDLE ), m_MinimumFreeMemoryMiB( 0 ), - m_ConsoleMode( false ) + m_ConsoleMode( false ), + m_PeriodicRestart( false ) { #ifdef __LINUX__ m_ConsoleMode = true; // Only console mode supported on Linux @@ -114,6 +115,11 @@ bool FBuildWorkerOptions::ProcessCommandLine( const AString & commandLine ) m_OverrideWorkMode = true; continue; } + else if ( token == "-periodicrestart" ) + { + m_PeriodicRestart = true; + continue; + } #if defined( __WINDOWS__ ) else if ( token.BeginsWith( "-minfreememory=" ) ) { @@ -175,6 +181,8 @@ void FBuildWorkerOptions::ShowUsageError() " Set minimum free memory (MiB) required to accept work.\n" " -nosubprocess\n" " (Windows) Don't spawn a sub-process worker copy.\n" + " -periodicrestart\n" + " Worker will restart every 4 hours.\n" "---------------------------------------------------------------------------\n" ; diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.h b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.h index c01ffe0b6..7f016555e 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.h +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.h @@ -39,6 +39,9 @@ class FBuildWorkerOptions // Console mode bool m_ConsoleMode; + // Other + bool m_PeriodicRestart; + private: void ShowUsageError(); }; diff --git a/Code/Tools/FBuild/FBuildWorker/Main.cpp b/Code/Tools/FBuild/FBuildWorker/Main.cpp index c4adb7cb3..262ce780b 100644 --- a/Code/Tools/FBuild/FBuildWorker/Main.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Main.cpp @@ -126,7 +126,7 @@ int Main( const AString & args ) // start the worker and wait for it to be closed int ret; { - Worker worker( args, options.m_ConsoleMode ); + Worker worker( args, options.m_ConsoleMode, options.m_PeriodicRestart ); if ( options.m_OverrideCPUAllocation ) { WorkerSettings::Get().SetNumCPUsToUse( options.m_CPUAllocation ); diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp index 5ffab8afd..3aaf8cfc2 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp @@ -28,6 +28,7 @@ #include "Core/FileIO/FileIO.h" #include "Core/Network/NetworkStartupHelper.h" #include "Core/Process/Process.h" +#include "Core/Process/Thread.h" #include "Core/Profile/Profile.h" #include "Core/Strings/AStackString.h" #include "Core/Tracing/Tracing.h" @@ -40,8 +41,9 @@ // CONSTRUCTOR //------------------------------------------------------------------------------ -Worker::Worker( const AString & args, bool consoleMode ) +Worker::Worker( const AString & args, bool consoleMode, bool periodicRestart ) : m_ConsoleMode( consoleMode ) + , m_PeriodicRestart( periodicRestart ) , m_MainWindow( nullptr ) , m_ConnectionPool( nullptr ) , m_NetworkStartupHelper( nullptr ) @@ -132,11 +134,8 @@ int32_t Worker::Work() } // spawn work thread - m_WorkThread = Thread::CreateThread( &WorkThreadWrapper, - "WorkerThread", - ( 256 * KILOBYTE ), - this ); - ASSERT( m_WorkThread != INVALID_THREAD_HANDLE ); + Thread workThread; + workThread.Start( &WorkThreadWrapper, "WorkerThread", this, ( 256 * KILOBYTE ) ); // Run the UI message loop if we're not in console mode if ( m_MainWindow ) @@ -145,7 +144,7 @@ int32_t Worker::Work() } // Join work thread and get exit code - return Thread::WaitForThread( m_WorkThread ); + return static_cast( workThread.Join() ); } // WorkThreadWrapper @@ -209,7 +208,7 @@ uint32_t Worker::WorkThread() UpdateUI(); - CheckForExeUpdate(); + CheckIfRestartNeeded(); PROFILE_SYNCHRONIZE @@ -434,9 +433,9 @@ void Worker::UpdateUI() m_UIUpdateTimer.Start(); } -// CheckForExeUpdate +// CheckIfRestartNeeded //------------------------------------------------------------------------------ -void Worker::CheckForExeUpdate() +void Worker::CheckIfRestartNeeded() { PROFILE_FUNCTION; @@ -457,6 +456,18 @@ void Worker::CheckForExeUpdate() return; // not running as a copy to allow restarts } + // Check if periodic restart time has been reached + if ( m_PeriodicRestart ) + { + const float periodicRestartSecs = ( 4.0f * 60.0f * 60.0f ); // 4 hours + if ( m_PeriodicRestartTimer.GetElapsed() > periodicRestartSecs ) + { + m_RestartNeeded = true; + JobQueueRemote::Get().SignalStopWorkers(); + return; + } + } + // get the current last write time const uint64_t lastWriteTime = FileIO::GetFileLastWriteTime( m_BaseExeName ); diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h index aa4c1bff6..afa86e2ba 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.h @@ -13,7 +13,6 @@ #include "Core/Containers/Singleton.h" #include "Core/Env/MSVCStaticAnalysis.h" #include "Core/FileIO/FileStream.h" -#include "Core/Process/Thread.h" // Forward Declarations //------------------------------------------------------------------------------ @@ -28,7 +27,7 @@ class WorkerSettings; class Worker : public Singleton { public: - explicit Worker( const AString & args, bool consoleMode ); + explicit Worker( const AString & args, bool consoleMode, bool periodicRestart ); ~Worker(); int32_t Work(); @@ -41,7 +40,7 @@ class Worker : public Singleton void UpdateAvailability(); void UpdateUI(); - void CheckForExeUpdate(); + void CheckIfRestartNeeded(); bool HasEnoughDiskSpace(); bool HasEnoughMemory(); @@ -51,6 +50,7 @@ class Worker : public Singleton void ErrorMessage( MSVC_SAL_PRINTF const char * fmtString, ... ) const FORMAT_STRING( 2, 3 ); bool m_ConsoleMode; + bool m_PeriodicRestart; WorkerWindow * m_MainWindow; Server * m_ConnectionPool; NetworkStartupHelper * m_NetworkStartupHelper; @@ -62,6 +62,7 @@ class Worker : public Singleton uint64_t m_LastWriteTime; bool m_WantToQuit; bool m_RestartNeeded; + Timer m_PeriodicRestartTimer; Timer m_UIUpdateTimer; FileStream m_TargetIncludeFolderLock; #if defined( __WINDOWS__ ) @@ -72,7 +73,6 @@ class Worker : public Singleton int32_t m_LastMemoryCheckResult; // -1 : No check done yet. 0=Not enough memory right now. 1=OK for now. #endif mutable AString m_LastStatusMessage; - Thread::ThreadHandle m_WorkThread; }; //------------------------------------------------------------------------------ diff --git a/Code/fbuild.bff b/Code/fbuild.bff index a9d91365c..d29eef84e 100644 --- a/Code/fbuild.bff +++ b/Code/fbuild.bff @@ -388,12 +388,18 @@ Settings .Targets_x64OSX_Debug = {} .Targets_x64OSX_Profile = {} .Targets_x64OSX_Release = {} -.Targets_ARMOSX_Debug = {} -.Targets_ARMOSX_Profile = {} -.Targets_ARMOSX_Release = {} +#if CLANG_SUPPORTS_ARMOSX + .Targets_ARMOSX_Debug = {} + .Targets_ARMOSX_Profile = {} + .Targets_ARMOSX_Release = {} + .Targets_OSX_Debug = {} + .Targets_OSX_Profile = {} + .Targets_OSX_Release = {} +#endif // External #include "..\External\LZ4\LZ4.bff" +#include "../External/xxHash/xxHash.bff" // Test Framework #include "TestFramework\TestFramework.bff" @@ -419,6 +425,14 @@ ForEach( .BuildConfig in .BuildConfigs ) Using( .BuildConfig ) Alias( 'All-$Platform$-$BuildConfigName$' ) { .Targets = .'Targets_$Platform$_$BuildConfigName$' } + + // Create additional Universal targets + #if CLANG_SUPPORTS_ARMOSX + If( .Platform == 'ARMOSX' ) + { + Alias( 'All-OSX-$BuildConfigName$' ) { .Targets = .'Targets_OSX_$BuildConfigName$' } + } + #endif } // Exes @@ -431,7 +445,18 @@ Alias( 'Exes' ) // Aliases : All-$Platform$ //------------------------------------------------------------------------------ -.Platforms = { 'x64', 'x64Clang', 'x64Linux', 'x64ClangLinux', 'x64OSX', 'ARMOSX' } +.Platforms = +{ + 'x64', + 'x64Clang', + 'x64Linux', + 'x64ClangLinux', + 'x64OSX' + #if CLANG_SUPPORTS_ARMOSX + 'ARMOSX' + 'OSX' + #endif +} .PlatformConfigs_x64 = { 'Debug', 'Analyze', 'Profile', 'Release', 'ASan', 'TSan' } .PlatformConfigs_x64Clang = { 'Debug', 'Analyze', 'Profile', 'Release', 'ASan', 'TSan' } .PlatformConfigs_x64Linux = { 'Debug', 'Profile', 'Release', 'ASan', 'TSan' } @@ -440,7 +465,10 @@ Alias( 'Exes' ) + { 'ASan', 'MSan', 'TSan' } #endif .PlatformConfigs_x64OSX = { 'Debug', 'Profile', 'Release' } -.PlatformConfigs_ARMOSX = { 'Debug', 'Profile', 'Release' } +#if CLANG_SUPPORTS_ARMOSX + .PlatformConfigs_ARMOSX = { 'Debug', 'Profile', 'Release' } + .PlatformConfigs_OSX = { 'Debug', 'Profile', 'Release' } +#endif ForEach( .Platform in .Platforms ) { Alias( 'All-$Platform$' ) @@ -479,9 +507,10 @@ ForEach( .Platform in .Platforms ) #if __OSX__ Alias( 'All' ) { - .Targets = { 'All-x64OSX' } - #if CLANG_SUPPORTS_OSXARM - + { 'All-ARMOSX' } + #if CLANG_SUPPORTS_ARMOSX + .Targets = { 'All-OSX' } // X64, ARM & Universal + #else + .Targets = { 'All-x64OSX' } // X64 only #endif } #endif @@ -628,7 +657,7 @@ Alias( 'All+Tests' ) .Folder_External = [ .Path = 'External' - .Projects = { 'LZ4-proj', 'SDKs-proj' } + .Projects = { 'LZ4-proj', 'SDKs-proj', 'xxHash-proj' } ] .Folder_Test = [ @@ -668,7 +697,8 @@ Alias( 'All+Tests' ) 'FBuildWorker-xcodeproj' 'LZ4-xcodeproj' 'OSUI-xcodeproj' - 'TestFramework-xcodeproj' } + 'TestFramework-xcodeproj' + 'xxHash-xcodeproj' } .ProjectConfigs = {} ForEach( .BuildConfig in .BuildConfigs ) { diff --git a/External/SDK/Clang/Clang.bff b/External/SDK/Clang/Clang.bff index 025ca4615..3cc75c65d 100644 --- a/External/SDK/Clang/Clang.bff +++ b/External/SDK/Clang/Clang.bff @@ -9,8 +9,8 @@ #define USING_CLANG_10 #endif #if __OSX__ && !CI_BUILD - #define USING_CLANG_8 - //#define USING_CLANG_12 + //#define USING_CLANG_8 + #define USING_CLANG_12 #endif #if __WINDOWS__ && !CI_BUILD //#define USING_CLANG_7 diff --git a/External/SDK/Clang/Linux/Clang10.bff b/External/SDK/Clang/Linux/Clang10.bff index be4ea90c7..fded8e6e3 100644 --- a/External/SDK/Clang/Linux/Clang10.bff +++ b/External/SDK/Clang/Linux/Clang10.bff @@ -31,7 +31,6 @@ Compiler( 'Compiler-Clang10' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/Linux/Clang3.bff b/External/SDK/Clang/Linux/Clang3.bff index 1d75392f9..efe3a08c7 100644 --- a/External/SDK/Clang/Linux/Clang3.bff +++ b/External/SDK/Clang/Linux/Clang3.bff @@ -28,7 +28,6 @@ Compiler( 'Compiler-Clang3' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/Linux/Clang6.bff b/External/SDK/Clang/Linux/Clang6.bff index 23eb30d9f..d55d81619 100644 --- a/External/SDK/Clang/Linux/Clang6.bff +++ b/External/SDK/Clang/Linux/Clang6.bff @@ -31,7 +31,6 @@ Compiler( 'Compiler-Clang6' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/Linux/Clang_CI.bff b/External/SDK/Clang/Linux/Clang_CI.bff index 926af79ab..971ea125f 100644 --- a/External/SDK/Clang/Linux/Clang_CI.bff +++ b/External/SDK/Clang/Linux/Clang_CI.bff @@ -30,7 +30,6 @@ Compiler( 'Compiler-Clang' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' diff --git a/External/SDK/Clang/OSX/Clang12.bff b/External/SDK/Clang/OSX/Clang12.bff index 5a042c8b4..6f3770dde 100644 --- a/External/SDK/Clang/OSX/Clang12.bff +++ b/External/SDK/Clang/OSX/Clang12.bff @@ -22,19 +22,15 @@ Compiler( 'Compiler-Clang12' ) // ToolChain //------------------------------------------------------------------------------ -.ToolChain_Clang_OSX = +.ToolChain_Clang_OSXCommon = [ - .Platform = 'x64OSX' - // Compiler Options .Compiler = 'Compiler-Clang12' .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output + ' -c' // Compile only + ' -g' // Generate debug info - + ' -arch x86_64' // Intel 64-bit + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' @@ -48,7 +44,6 @@ Compiler( 'Compiler-Clang12' ) + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types + ' -Wno-implicit-exception-spec-mismatch' // Fires on our new/delete operator (Clang bug?) - .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' .CompilerOptionsC = ' -x c $CommonCompilerOptions$' @@ -58,7 +53,7 @@ Compiler( 'Compiler-Clang12' ) // Linker .Linker = '$Clang12_BasePath$/clang++' - .LinkerOptions = '"%1" -o "%2" -g -arch x86_64' + .LinkerOptions = '"%1" -o "%2" -g' // File Extensions .LibExtension = '.a' @@ -68,50 +63,26 @@ Compiler( 'Compiler-Clang12' ) .UseExceptions = ' -fexceptions' ] +.ToolChain_Clang_OSX = +[ + Using( .ToolChain_Clang_OSXCommon ) + + // Intel 64-bit + .Platform = 'x64OSX' + .CompilerOptions + ' -arch x86_64' + .CompilerOptionsC + ' -arch x86_64' + .LinkerOptions + ' -arch x86_64' +] + .ToolChain_Clang_ARMOSX = [ + Using( .ToolChain_Clang_OSXCommon ) + + // Apple ARM Silicon .Platform = 'ARMOSX' - - // Compiler Options - .Compiler = 'Compiler-Clang12' - .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output - + ' -c' // Compile only - + ' -g' // Generate debug info - + ' -arch arm64' // Apple Silicon - + ' -D__OSX__' // Platform define - + ' -D__APPLE__' // Platform define - + ' -D__ARM64__' // Architecture define - - // Include paths - + ' -I./' - - // Enable warnings - + ' -Wall -Werror -Wfatal-errors' // warnings as errors - + ' -Wextra' - - // Disabled warnings - + ' -Wno-#pragma-messages' - + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types - + ' -Wno-implicit-exception-spec-mismatch' // Fires on our new/delete operator (Clang bug?) - - - .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' - .CompilerOptionsC = ' -x c $CommonCompilerOptions$' - - // Librarian - .Librarian = '$Clang12_BasePath$/ar' - .LibrarianOptions = 'rcs "%2" "%1"' - - // Linker - .Linker = '$Clang12_BasePath$/clang++' - .LinkerOptions = '"%1" -o "%2" -g -arch arm64' - - // File Extensions - .LibExtension = '.a' - .ExeExtension = '' - - // Exception Control - .UseExceptions = ' -fexceptions' + .CompilerOptions + ' -arch arm64' + .CompilerOptionsC + ' -arch arm64' + .LinkerOptions + ' -arch arm64' ] //------------------------------------------------------------------------------ diff --git a/External/SDK/Clang/OSX/Clang8.bff b/External/SDK/Clang/OSX/Clang8.bff index bed1ff306..c5ea4497d 100644 --- a/External/SDK/Clang/OSX/Clang8.bff +++ b/External/SDK/Clang/OSX/Clang8.bff @@ -32,7 +32,6 @@ Compiler( 'Compiler-Clang8' ) + ' -m64' // x86_64 + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define - + ' -D__X64__' // Architecture define + ' -mmacosx-version-min=10.7' + ' -stdlib=libc++' diff --git a/External/SDK/Clang/OSX/Clang_CI.bff b/External/SDK/Clang/OSX/Clang_CI.bff index d987a23c4..747c87f22 100644 --- a/External/SDK/Clang/OSX/Clang_CI.bff +++ b/External/SDK/Clang/OSX/Clang_CI.bff @@ -20,19 +20,15 @@ Compiler( 'Compiler-Clang12' ) // ToolChain //------------------------------------------------------------------------------ -.ToolChain_Clang_OSX = +.ToolChain_Clang_OSXCommon = [ - .Platform = 'x64OSX' - // Compiler Options .Compiler = 'Compiler-Clang12' .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output + ' -c' // Compile only + ' -g' // Generate debug info - + ' -arch x86_64' // Intel 64-bit + ' -D__OSX__' // Platform define + ' -D__APPLE__' // Platform define - + ' -D__X64__' // Architecture define // Include paths + ' -I./' @@ -56,7 +52,7 @@ Compiler( 'Compiler-Clang12' ) // Linker .Linker = 'CLANGXX_BINARY' - .LinkerOptions = '"%1" -o "%2" -g -arch x86_64' + .LinkerOptions = '"%1" -o "%2" -g' // File Extensions .LibExtension = '.a' @@ -66,50 +62,26 @@ Compiler( 'Compiler-Clang12' ) .UseExceptions = ' -fexceptions' ] +.ToolChain_Clang_OSX = +[ + Using( .ToolChain_Clang_OSXCommon ) + + // Intel 64-bit + .Platform = 'x64OSX' + .CompilerOptions + ' -arch x86_64' + .CompilerOptionsC + ' -arch x86_64' + .LinkerOptions + ' -arch x86_64' +] + .ToolChain_Clang_ARMOSX = [ + Using( .ToolChain_Clang_OSXCommon ) + + // Apple ARM Silicon .Platform = 'ARMOSX' - - // Compiler Options - .Compiler = 'Compiler-Clang12' - .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output - + ' -c' // Compile only - + ' -g' // Generate debug info - + ' -arch arm64' // Apple Silicon - + ' -D__OSX__' // Platform define - + ' -D__APPLE__' // Platform define - + ' -D__ARM64__' // Architecture define - - // Include paths - + ' -I./' - - // Enable warnings - + ' -Wall -Werror -Wfatal-errors' // warnings as errors - + ' -Wextra' - - // Disabled warnings - + ' -Wno-#pragma-messages' - + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types - + ' -Wno-implicit-exception-spec-mismatch' // Fires on our new/delete operator (Clang bug?) - - - .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' - .CompilerOptionsC = ' -x c $CommonCompilerOptions$' - - // Librarian - .Librarian = '/usr/bin/ar' - .LibrarianOptions = 'rcs "%2" "%1"' - - // Linker - .Linker = 'CLANGXX_BINARY' - .LinkerOptions = '"%1" -o "%2" -g -arch arm64' - - // File Extensions - .LibExtension = '.a' - .ExeExtension = '' - - // Exception Control - .UseExceptions = ' -fexceptions' + .CompilerOptions + ' -arch arm64' + .CompilerOptionsC + ' -arch arm64' + .LinkerOptions + ' -arch arm64' ] //------------------------------------------------------------------------------ diff --git a/External/SDK/GCC/Linux/GCC4.bff b/External/SDK/GCC/Linux/GCC4.bff index a271f15c8..bdcb3c9a3 100644 --- a/External/SDK/GCC/Linux/GCC4.bff +++ b/External/SDK/GCC/Linux/GCC4.bff @@ -33,7 +33,6 @@ Compiler( 'Compiler-GCC4' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h // Include paths diff --git a/External/SDK/GCC/Linux/GCC7.bff b/External/SDK/GCC/Linux/GCC7.bff index e1ea0a313..01628903a 100644 --- a/External/SDK/GCC/Linux/GCC7.bff +++ b/External/SDK/GCC/Linux/GCC7.bff @@ -33,7 +33,6 @@ Compiler( 'Compiler-GCC7' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h // Include paths diff --git a/External/SDK/GCC/Linux/GCC9.bff b/External/SDK/GCC/Linux/GCC9.bff index b42eabf25..677df0531 100644 --- a/External/SDK/GCC/Linux/GCC9.bff +++ b/External/SDK/GCC/Linux/GCC9.bff @@ -33,7 +33,6 @@ Compiler( 'Compiler-GCC9' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h // Include paths diff --git a/External/SDK/GCC/Linux/GCC_CI.bff b/External/SDK/GCC/Linux/GCC_CI.bff index 511e99cc9..7168509b1 100644 --- a/External/SDK/GCC/Linux/GCC_CI.bff +++ b/External/SDK/GCC/Linux/GCC_CI.bff @@ -32,7 +32,6 @@ Compiler( 'Compiler-GCC' ) + ' -g' // Generate debug info + ' -m64' // x86_64 + ' -D__LINUX__' // Platform define - + ' -D__X64__' // Architecture define + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h // Include paths diff --git a/External/SDK/Windows/Windows10SDK.bff b/External/SDK/Windows/Windows10SDK.bff index 1135b040d..9e987ece1 100644 --- a/External/SDK/Windows/Windows10SDK.bff +++ b/External/SDK/Windows/Windows10SDK.bff @@ -61,7 +61,6 @@ .CommonCompilerOptions = .WindowsSDK_IncludePaths + .WindowsSDK_Defines + ' -DWIN64' - + ' -D__X64__' .CompilerOptions = .CommonCompilerOptions .CompilerOptionsC = .CommonCompilerOptions @@ -85,7 +84,6 @@ .CommonCompilerOptions = .WindowsSDK_IncludePaths + .WindowsSDK_Defines + ' -DWIN64' - + ' -D__X64__' .CompilerOptions = .CommonCompilerOptions .CompilerOptionsC = .CommonCompilerOptions diff --git a/External/xxHash/0.8.1/CHANGELOG b/External/xxHash/0.8.1/CHANGELOG new file mode 100644 index 000000000..ff59d8bb1 --- /dev/null +++ b/External/xxHash/0.8.1/CHANGELOG @@ -0,0 +1,71 @@ +v0.8.1 +- perf : much improved performance for XXH3 streaming variants, notably on gcc and msvc +- perf : improved XXH64 speed and latency on small inputs +- perf : small XXH32 speed and latency improvement on small inputs of random size +- perf : minor stack usage improvement for XXH32 and XXH64 +- api : new experimental variants XXH3_*_withSecretandSeed() +- api : update XXH3_generateSecret(), can no generate secret of any size (>= XXH3_SECRET_SIZE_MIN) +- cli : xxhsum can now generate and check XXH3 checksums, using command `-H3` +- build: can build xxhash without XXH3, with new build macro XXH_NO_XXH3 +- build: fix xxh_x86dispatch build with MSVC, by @apankrat +- build: XXH_INLINE_ALL can always be used safely, even after XXH_NAMESPACE or a previous XXH_INLINE_ALL +- build: improved PPC64LE vector support, by @mpe +- install: fix pkgconfig, by @ellert +- install: compatibility with Haiku, by @Begasus +- doc : code comments made compatible with doxygen, by @easyaspi314 +- misc : XXH_ACCEPT_NULL_INPUT_POINTER is no longer necessary, all functions can accept NULL input pointers, as long as size == 0 +- misc : complete refactor of CI tests on Github Actions, offering much larger coverage, by @t-mat +- misc : xxhsum code base split into multiple specialized units, within directory cli/, by @easyaspi314 + +v0.8.0 +- api : stabilize XXH3 +- cli : xxhsum can parse BSD-style --check lines, by @WayneD +- cli : `xxhsum -` accepts console input, requested by @jaki +- cli : xxhsum accepts -- separator, by @jaki +- cli : fix : print correct default algo for symlinked helpers, by @martinetd +- install: improved pkgconfig script, allowing custom install locations, requested by @ellert + +v0.7.4 +- perf: automatic vector detection and selection at runtime (`xxh_x86dispatch.h`), initiated by @easyaspi314 +- perf: added AVX512 support, by @gzm55 +- api : new: secret generator `XXH_generateSecret()`, suggested by @koraa +- api : fix: XXH3_state_t is movable, identified by @koraa +- api : fix: state is correctly aligned in AVX mode (unlike `malloc()`), by @easyaspi314 +- api : fix: streaming generated wrong values in some combination of random ingestion lengths, reported by @WayneD +- cli : fix unicode print on Windows, by @easyaspi314 +- cli : can `-c` check file generated by sfv +- build: `make DISPATCH=1` generates `xxhsum` and `libxxhash` with runtime vector detection (x86/x64 only) +- install: cygwin installation support +- doc : Cryptol specification of XXH32 and XXH64, by @weaversa + +v0.7.3 +- perf: improved speed for large inputs (~+20%) +- perf: improved latency for small inputs (~10%) +- perf: s390x Vectorial code, by @easyaspi314 +- cli: improved support for Unicode filenames on Windows, thanks to @easyaspi314 and @t-mat +- api: `xxhash.h` can now be included in any order, with and without `XXH_STATIC_LINKING_ONLY` and `XXH_INLINE_ALL` +- build: xxHash's implementation transferred into `xxhash.h`. No more need to have `xxhash.c` in the `/include` directory for `XXH_INLINE_ALL` to work +- install: created pkg-config file, by @bket +- install: VCpkg installation instructions, by @LilyWangL +- doc: Highly improved code documentation, by @easyaspi314 +- misc: New test tool in `/tests/collisions`: brute force collision tester for 64-bit hashes + +v0.7.2 +- Fixed collision ratio of `XXH128` for some specific input lengths, reported by @svpv +- Improved `VSX` and `NEON` variants, by @easyaspi314 +- Improved performance of scalar code path (`XXH_VECTOR=0`), by @easyaspi314 +- `xxhsum`: can generate 128-bit hashes with the `-H2` option (note: for experimental purposes only! `XXH128` is not yet frozen) +- `xxhsum`: option `-q` removes status notifications + +v0.7.1 +- Secret first: the algorithm computation can be altered by providing a "secret", which is any blob of bytes, of size >= `XXH3_SECRET_SIZE_MIN`. +- `seed` is still available, and acts as a secret generator +- updated `ARM NEON` variant by @easyaspi314 +- Streaming implementation is available +- Improve compatibility and performance with Visual Studio, with help from @aras-p +- Better integration when using `XXH_INLINE_ALL`: do not pollute host namespace, use its own macros, such as `XXH_ASSERT()`, `XXH_ALIGN`, etc. +- 128-bit variant provides helper functions for comparison of hashes. +- Better `clang` generation of `rotl` instruction, thanks to @easyaspi314 +- `XXH_REROLL` build macro to reduce binary size, by @easyaspi314 +- Improved `cmake` script, by @Mezozoysky +- Full benchmark program provided in `/tests/bench` diff --git a/External/xxHash/0.8.1/LICENSE b/External/xxHash/0.8.1/LICENSE new file mode 100644 index 000000000..6bc30a1bc --- /dev/null +++ b/External/xxHash/0.8.1/LICENSE @@ -0,0 +1,26 @@ +xxHash Library +Copyright (c) 2012-2020 Yann Collet +All rights reserved. + +BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/External/xxHash/0.8.1/README.md b/External/xxHash/0.8.1/README.md new file mode 100644 index 000000000..2406c8d24 --- /dev/null +++ b/External/xxHash/0.8.1/README.md @@ -0,0 +1,236 @@ + +xxHash - Extremely fast hash algorithm +====================================== + +xxHash is an Extremely fast Hash algorithm, running at RAM speed limits. +It successfully completes the [SMHasher](https://code.google.com/p/smhasher/wiki/SMHasher) test suite +which evaluates collision, dispersion and randomness qualities of hash functions. +Code is highly portable, and hashes are identical across all platforms (little / big endian). + +|Branch |Status | +|------------|---------| +|dev | [![Build Status](https://github.com/Cyan4973/xxHash/actions/workflows/ci.yml/badge.svg?branch=dev)](https://github.com/Cyan4973/xxHash/actions?query=branch%3Adev+) | + + +Benchmarks +------------------------- + +The reference system uses an Intel i7-9700K cpu, and runs Ubuntu x64 20.04. +The [open source benchmark program] is compiled with `clang` v10.0 using `-O3` flag. + +| Hash Name | Width | Bandwidth (GB/s) | Small Data Velocity | Quality | Comment | +| --------- | ----- | ---------------- | ----- | --- | --- | +| __XXH3__ (SSE2) | 64 | 31.5 GB/s | 133.1 | 10 +| __XXH128__ (SSE2) | 128 | 29.6 GB/s | 118.1 | 10 +| _RAM sequential read_ | N/A | 28.0 GB/s | N/A | N/A | _for reference_ +| City64 | 64 | 22.0 GB/s | 76.6 | 10 +| T1ha2 | 64 | 22.0 GB/s | 99.0 | 9 | Slightly worse [collisions] +| City128 | 128 | 21.7 GB/s | 57.7 | 10 +| __XXH64__ | 64 | 19.4 GB/s | 71.0 | 10 +| SpookyHash | 64 | 19.3 GB/s | 53.2 | 10 +| Mum | 64 | 18.0 GB/s | 67.0 | 9 | Slightly worse [collisions] +| __XXH32__ | 32 | 9.7 GB/s | 71.9 | 10 +| City32 | 32 | 9.1 GB/s | 66.0 | 10 +| Murmur3 | 32 | 3.9 GB/s | 56.1 | 10 +| SipHash | 64 | 3.0 GB/s | 43.2 | 10 +| FNV64 | 64 | 1.2 GB/s | 62.7 | 5 | Poor avalanche properties +| Blake2 | 256 | 1.1 GB/s | 5.1 | 10 | Cryptographic +| SHA1 | 160 | 0.8 GB/s | 5.6 | 10 | Cryptographic but broken +| MD5 | 128 | 0.6 GB/s | 7.8 | 10 | Cryptographic but broken + +[open source benchmark program]: https://github.com/Cyan4973/xxHash/tree/release/tests/bench +[collisions]: https://github.com/Cyan4973/xxHash/wiki/Collision-ratio-comparison#collision-study + +note 1: Small data velocity is a _rough_ evaluation of algorithm's efficiency on small data. For more detailed analysis, please refer to next paragraph. + +note 2: some algorithms feature _faster than RAM_ speed. In which case, they can only reach their full speed when input data is already in CPU cache (L3 or better). Otherwise, they max out on RAM speed limit. + +### Small data + +Performance on large data is only one part of the picture. +Hashing is also very useful in constructions like hash tables and bloom filters. +In these use cases, it's frequent to hash a lot of small data (starting at a few bytes). +Algorithm's performance can be very different for such scenarios, since parts of the algorithm, +such as initialization or finalization, become fixed cost. +The impact of branch mis-prediction also becomes much more present. + +XXH3 has been designed for excellent performance on both long and small inputs, +which can be observed in the following graph: + +![XXH3, latency, random size](https://user-images.githubusercontent.com/750081/61976089-aedeab00-af9f-11e9-9239-e5375d6c080f.png) + +For a more detailed analysis, visit the wiki : +https://github.com/Cyan4973/xxHash/wiki/Performance-comparison#benchmarks-concentrating-on-small-data- + +Quality +------------------------- + +Speed is not the only property that matters. +Produced hash values must respect excellent dispersion and randomness properties, +so that any sub-section of it can be used to maximally spread out a table or index, +as well as reduce the amount of collisions to the minimal theoretical level, following the [birthday paradox]. + +`xxHash` has been tested with Austin Appleby's excellent SMHasher test suite, +and passes all tests, ensuring reasonable quality levels. +It also passes extended tests from [newer forks of SMHasher], featuring additional scenarios and conditions. + +Finally, xxHash provides its own [massive collision tester](https://github.com/Cyan4973/xxHash/tree/dev/tests/collisions), +able to generate and compare billions of hashes to test the limits of 64-bit hash algorithms. +On this front too, xxHash features good results, in line with the [birthday paradox]. +A more detailed analysis is documented [in the wiki](https://github.com/Cyan4973/xxHash/wiki/Collision-ratio-comparison). + +[birthday paradox]: https://en.wikipedia.org/wiki/Birthday_problem +[newer forks of SMHasher]: https://github.com/rurban/smhasher + + +### Build modifiers + +The following macros can be set at compilation time to modify libxxhash's behavior. They are generally disabled by default. + +- `XXH_INLINE_ALL`: Make all functions `inline`, with implementations being directly included within `xxhash.h`. + Inlining functions is beneficial for speed on small keys. + It's _extremely effective_ when key length is expressed as _a compile time constant_, + with performance improvements observed in the +200% range . + See [this article](https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html) for details. +- `XXH_PRIVATE_API`: same outcome as `XXH_INLINE_ALL`. Still available for legacy support. + The name underlines that `XXH_*` symbols will not be exported. +- `XXH_NAMESPACE`: Prefixes all symbols with the value of `XXH_NAMESPACE`. + This macro can only use compilable character set. + Useful to evade symbol naming collisions, + in case of multiple inclusions of xxHash's source code. + Client applications still use the regular function names, + as symbols are automatically translated through `xxhash.h`. +- `XXH_FORCE_MEMORY_ACCESS`: The default method `0` uses a portable `memcpy()` notation. + Method `1` uses a gcc-specific `packed` attribute, which can provide better performance for some targets. + Method `2` forces unaligned reads, which is not standards compliant, but might sometimes be the only way to extract better read performance. + Method `3` uses a byteshift operation, which is best for old compilers which don't inline `memcpy()` or big-endian systems without a byteswap instruction +- `XXH_FORCE_ALIGN_CHECK`: Use a faster direct read path when input is aligned. + This option can result in dramatic performance improvement when input to hash is aligned on 32 or 64-bit boundaries, + when running on architectures unable to load memory from unaligned addresses, or suffering a performance penalty from it. + It is (slightly) detrimental on platform with good unaligned memory access performance (same instruction for both aligned and unaligned accesses). + This option is automatically disabled on `x86`, `x64` and `aarch64`, and enabled on all other platforms. +- `XXH_VECTOR` : manually select a vector instruction set (default: auto-selected at compilation time). Available instruction sets are `XXH_SCALAR`, `XXH_SSE2`, `XXH_AVX2`, `XXH_AVX512`, `XXH_NEON` and `XXH_VSX`. Compiler may require additional flags to ensure proper support (for example, `gcc` on linux will require `-mavx2` for AVX2, and `-mavx512f` for AVX512). +- `XXH_NO_PREFETCH` : disable prefetching. Some platforms or situations may perform better without prefetching. XXH3 only. +- `XXH_PREFETCH_DIST` : select prefetching distance. For close-to-metal adaptation to specific hardware platforms. XXH3 only. +- `XXH_NO_INLINE_HINTS`: By default, xxHash uses `__attribute__((always_inline))` and `__forceinline` to improve performance at the cost of code size. + Defining this macro to 1 will mark all internal functions as `static`, allowing the compiler to decide whether to inline a function or not. + This is very useful when optimizing for smallest binary size, + and is automatically defined when compiling with `-O0`, `-Os`, `-Oz`, or `-fno-inline` on GCC and Clang. + This may also increase performance depending on compiler and architecture. +- `XXH32_ENDJMP`: Switch multi-branch finalization stage of XXH32 by a single jump. + This is generally undesirable for performance, especially when hashing inputs of random sizes. + But depending on exact architecture and compiler, a jump might provide slightly better performance on small inputs. Disabled by default. +- `XXH_STATIC_LINKING_ONLY`: gives access to internal state declaration, required for static allocation. + Incompatible with dynamic linking, due to risks of ABI changes. +- `XXH_NO_XXH3` : removes symbols related to `XXH3` (both 64 & 128 bits) from generated binary. + Useful to reduce binary size, notably for applications which do not use `XXH3`. +- `XXH_NO_LONG_LONG`: removes compilation of algorithms relying on 64-bit types (XXH3 and XXH64). Only XXH32 will be compiled. + Useful for targets (architectures and compilers) without 64-bit support. +- `XXH_IMPORT`: MSVC specific: should only be defined for dynamic linking, as it prevents linkage errors. +- `XXH_CPU_LITTLE_ENDIAN`: By default, endianness is determined by a runtime test resolved at compile time. + If, for some reason, the compiler cannot simplify the runtime test, it can cost performance. + It's possible to skip auto-detection and simply state that the architecture is little-endian by setting this macro to 1. + Setting it to 0 states big-endian. +- `XXH_DEBUGLEVEL` : When set to any value >= 1, enables `assert()` statements. + This (slightly) slows down execution, but may help finding bugs during debugging sessions. + +When compiling the Command Line Interface `xxhsum` with `make`, the following environment variables can also be set : +- `DISPATCH=1` : use `xxh_x86dispatch.c`, to automatically select between `scalar`, `sse2`, `avx2` or `avx512` instruction set at runtime, depending on local host. This option is only valid for `x86`/`x64` systems. + + +### Building xxHash - Using vcpkg + +You can download and install xxHash using the [vcpkg](https://github.com/Microsoft/vcpkg) dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install xxhash + +The xxHash port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + + +### Example + +The simplest example calls xxhash 64-bit variant as a one-shot function +generating a hash value from a single buffer, and invoked from a C/C++ program: + +```C +#include "xxhash.h" + + (...) + XXH64_hash_t hash = XXH64(buffer, size, seed); +} +``` + +Streaming variant is more involved, but makes it possible to provide data incrementally: + +```C +#include "stdlib.h" /* abort() */ +#include "xxhash.h" + + +XXH64_hash_t calcul_hash_streaming(FileHandler fh) +{ + /* create a hash state */ + XXH64_state_t* const state = XXH64_createState(); + if (state==NULL) abort(); + + size_t const bufferSize = SOME_SIZE; + void* const buffer = malloc(bufferSize); + if (buffer==NULL) abort(); + + /* Initialize state with selected seed */ + XXH64_hash_t const seed = 0; /* or any other value */ + if (XXH64_reset(state, seed) == XXH_ERROR) abort(); + + /* Feed the state with input data, any size, any number of times */ + (...) + while ( /* some data left */ ) { + size_t const length = get_more_data(buffer, bufferSize, fh); + if (XXH64_update(state, buffer, length) == XXH_ERROR) abort(); + (...) + } + (...) + + /* Produce the final hash value */ + XXH64_hash_t const hash = XXH64_digest(state); + + /* State could be re-used; but in this example, it is simply freed */ + free(buffer); + XXH64_freeState(state); + + return hash; +} +``` + + +### License + +The library files `xxhash.c` and `xxhash.h` are BSD licensed. +The utility `xxhsum` is GPL licensed. + + +### Other programming languages + +Beyond the C reference version, +xxHash is also available from many different programming languages, +thanks to great contributors. +They are [listed here](http://www.xxhash.com/#other-languages). + + +### Packaging status + +Many distributions bundle a package manager +which allows easy xxhash installation as both a `libxxhash` library +and `xxhsum` command line interface. + +[![Packaging status](https://repology.org/badge/vertical-allrepos/xxhash.svg)](https://repology.org/project/xxhash/versions) + + +### Special Thanks + +- Takayuki Matsuoka, aka @t-mat, for creating `xxhsum -c` and great support during early xxh releases +- Mathias Westerdahl, aka @JCash, for introducing the first version of `XXH64` +- Devin Hussey, aka @easyaspi314, for incredible low-level optimizations on `XXH3` and `XXH128` diff --git a/External/xxHash/0.8.1/libxxhash.pc.in b/External/xxHash/0.8.1/libxxhash.pc.in new file mode 100644 index 000000000..28c164485 --- /dev/null +++ b/External/xxHash/0.8.1/libxxhash.pc.in @@ -0,0 +1,15 @@ +# xxHash - Extremely fast hash algorithm +# Copyright (C) 2012-2020, Yann Collet, Facebook +# BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +prefix=@PREFIX@ +exec_prefix=@EXECPREFIX@ +includedir=${prefix}/@INCLUDEDIR@ +libdir=${exec_prefix}/@LIBDIR@ + +Name: xxhash +Description: extremely fast hash algorithm +URL: http://www.xxhash.com/ +Version: @VERSION@ +Libs: -L${libdir} -lxxhash +Cflags: -I${includedir} diff --git a/External/xxHash/0.8.1/xxh3.h b/External/xxHash/0.8.1/xxh3.h new file mode 100644 index 000000000..f7dc1959b --- /dev/null +++ b/External/xxHash/0.8.1/xxh3.h @@ -0,0 +1,55 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Development source file for `xxh3` + * Copyright (C) 2019-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/* + * Note: This file used to host the source code of XXH3_* variants. + * during the development period. + * The source code is now properly integrated within xxhash.h. + * + * xxh3.h is no longer useful, + * but it is still provided for compatibility with source code + * which used to include it directly. + * + * Programs are now highly discouraged to include xxh3.h. + * Include `xxhash.h` instead, which is the officially supported interface. + * + * In the future, xxh3.h will start to generate warnings, then errors, + * then it will be removed from source package and from include directory. + */ + +/* Simulate the same impact as including the old xxh3.h source file */ + +#define XXH_INLINE_ALL +#include "xxhash.h" diff --git a/External/xxHash/0.8.1/xxh_x86dispatch.c b/External/xxHash/0.8.1/xxh_x86dispatch.c new file mode 100644 index 000000000..399bad904 --- /dev/null +++ b/External/xxHash/0.8.1/xxh_x86dispatch.c @@ -0,0 +1,770 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (C) 2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + + +/*! + * @file xxh_x86dispatch.c + * + * Automatic dispatcher code for the @ref xxh3_family on x86-based targets. + * + * Optional add-on. + * + * **Compile this file with the default flags for your target.** Do not compile + * with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`, there will be + * an error. See @ref XXH_X86DISPATCH_ALLOW_AVX for details. + * + * @defgroup dispatch x86 Dispatcher + * @{ + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)) +# error "Dispatching is currently only supported on x86 and x86_64." +#endif + +/*! + * @def XXH_X86DISPATCH_ALLOW_AVX + * @brief Disables the AVX sanity check. + * + * Don't compile xxh_x86dispatch.c with options like `-mavx*`, `-march=native`, + * or `/arch:AVX*`. It is intended to be compiled for the minimum target, and + * it selectively enables SSE2, AVX2, and AVX512 when it is needed. + * + * Using this option _globally_ allows this feature, and therefore makes it + * undefined behavior to execute on any CPU without said feature. + * + * Even if the source code isn't directly using AVX intrinsics in a function, + * the compiler can still generate AVX code from autovectorization and by + * "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128). + * + * Use the same flags that you use to compile the rest of the program; this + * file will safely generate SSE2, AVX2, and AVX512 without these flags. + * + * Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check, and feel free to open + * an issue if there is a target in the future where AVX is a default feature. + */ +#ifdef XXH_DOXYGEN +# define XXH_X86DISPATCH_ALLOW_AVX +#endif + +#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX) +# error "Do not compile xxh_x86dispatch.c with AVX enabled! See the comment above." +#endif + +#ifdef __has_include +# define XXH_HAS_INCLUDE(header) __has_include(header) +#else +# define XXH_HAS_INCLUDE(header) 0 +#endif + +/*! + * @def XXH_DISPATCH_SCALAR + * @brief Enables/dispatching the scalar code path. + * + * If this is defined to 0, SSE2 support is assumed. This reduces code size + * when the scalar path is not needed. + * + * This is automatically defined to 0 when... + * - SSE2 support is enabled in the compiler + * - Targeting x86_64 + * - Targeting Android x86 + * - Targeting macOS + */ +#ifndef XXH_DISPATCH_SCALAR +# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \ + || defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \ + || defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */ +# define XXH_DISPATCH_SCALAR 0 /* disable */ +# else +# define XXH_DISPATCH_SCALAR 1 +# endif +#endif +/*! + * @def XXH_DISPATCH_AVX2 + * @brief Enables/disables dispatching for AVX2. + * + * This is automatically detected if it is not defined. + * - GCC 4.7 and later are known to support AVX2, but >4.9 is required for + * to get the AVX2 intrinsics and typedefs without -mavx -mavx2. + * - Visual Studio 2013 Update 2 and later are known to support AVX2. + * - The GCC/Clang internal header `` is detected. While this is + * not allowed to be included directly, it still appears in the builtin + * include path and is detectable with `__has_include`. + * + * @see XXH_AVX2 + */ +#ifndef XXH_DISPATCH_AVX2 +# if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \ + || (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \ + || (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \ + || XXH_HAS_INCLUDE() /* GCC/Clang internal header */ +# define XXH_DISPATCH_AVX2 1 /* enable dispatch towards AVX2 */ +# else +# define XXH_DISPATCH_AVX2 0 +# endif +#endif /* XXH_DISPATCH_AVX2 */ + +/*! + * @def XXH_DISPATCH_AVX512 + * @brief Enables/disables dispatching for AVX512. + * + * Automatically detected if one of the following conditions is met: + * - GCC 4.9 and later are known to support AVX512. + * - Visual Studio 2017 and later are known to support AVX2. + * - The GCC/Clang internal header `` is detected. While this + * is not allowed to be included directly, it still appears in the builtin + * include path and is detectable with `__has_include`. + * + * @see XXH_AVX512 + */ +#ifndef XXH_DISPATCH_AVX512 +# if (defined(__GNUC__) \ + && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \ + || (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \ + || XXH_HAS_INCLUDE() /* GCC/Clang internal header */ +# define XXH_DISPATCH_AVX512 1 /* enable dispatch towards AVX512 */ +# else +# define XXH_DISPATCH_AVX512 0 +# endif +#endif /* XXH_DISPATCH_AVX512 */ + +/*! + * @def XXH_TARGET_SSE2 + * @brief Allows a function to be compiled with SSE2 intrinsics. + * + * Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used + * even with `-mno-sse2`. + * + * @def XXH_TARGET_AVX2 + * @brief Like @ref XXH_TARGET_SSE2, but for AVX2. + * + * @def XXH_TARGET_AVX512 + * @brief Like @ref XXH_TARGET_SSE2, but for AVX512. + */ +#if defined(__GNUC__) +# include /* SSE2 */ +# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 +# include /* AVX2, AVX512F */ +# endif +# define XXH_TARGET_SSE2 __attribute__((__target__("sse2"))) +# define XXH_TARGET_AVX2 __attribute__((__target__("avx2"))) +# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f"))) +#elif defined(_MSC_VER) +# include +# define XXH_TARGET_SSE2 +# define XXH_TARGET_AVX2 +# define XXH_TARGET_AVX512 +#else +# error "Dispatching is currently not supported for your compiler." +#endif + +#ifdef XXH_DISPATCH_DEBUG +/* debug logging */ +# include +# define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); } +#else +# define XXH_debugPrint(str) ((void)0) +# undef NDEBUG /* avoid redefinition */ +# define NDEBUG +#endif +#include + +#define XXH_INLINE_ALL +#define XXH_X86DISPATCH +#include "xxhash.h" + +/* + * Support both AT&T and Intel dialects + * + * GCC doesn't convert AT&T syntax to Intel syntax, and will error out if + * compiled with -masm=intel. Instead, it supports dialect switching with + * curly braces: { AT&T syntax | Intel syntax } + * + * Clang's integrated assembler automatically converts AT&T syntax to Intel if + * needed, making the dialect switching useless (it isn't even supported). + * + * Note: Comments are written in the inline assembly itself. + */ +#ifdef __clang__ +# define XXH_I_ATT(intel, att) att "\n\t" +#else +# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t" +#endif + +/*! + * @internal + * @brief Runs CPUID. + * + * @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively. + * @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }` + */ +static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd) +{ +#if defined(_MSC_VER) + __cpuidex(abcd, eax, ecx); +#else + xxh_u32 ebx, edx; +# if defined(__i386__) && defined(__PIC__) + __asm__( + "# Call CPUID\n\t" + "#\n\t" + "# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t" + "# EBX, so we use EDI instead.\n\t" + XXH_I_ATT("mov edi, ebx", "movl %%ebx, %%edi") + XXH_I_ATT("cpuid", "cpuid" ) + XXH_I_ATT("xchg edi, ebx", "xchgl %%ebx, %%edi") + : "=D" (ebx), +# else + __asm__( + "# Call CPUID\n\t" + XXH_I_ATT("cpuid", "cpuid") + : "=b" (ebx), +# endif + "+a" (eax), "+c" (ecx), "=d" (edx)); + abcd[0] = eax; + abcd[1] = ebx; + abcd[2] = ecx; + abcd[3] = edx; +#endif +} + +/* + * Modified version of Intel's guide + * https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family + */ + +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 +/*! + * @internal + * @brief Runs `XGETBV`. + * + * While the CPU may support AVX2, the operating system might not properly save + * the full YMM/ZMM registers. + * + * xgetbv is used for detecting this: Any compliant operating system will define + * a set of flags in the xcr0 register indicating how it saves the AVX registers. + * + * You can manually disable this flag on Windows by running, as admin: + * + * bcdedit.exe /set xsavedisable 1 + * + * and rebooting. Run the same command with 0 to re-enable it. + */ +static xxh_u64 XXH_xgetbv(void) +{ +#if defined(_MSC_VER) + return _xgetbv(0); /* min VS2010 SP1 compiler is required */ +#else + xxh_u32 xcr0_lo, xcr0_hi; + __asm__( + "# Call XGETBV\n\t" + "#\n\t" + "# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t" + "# the XGETBV opcode, so we encode it by hand instead.\n\t" + "# See for details.\n\t" + ".byte 0x0f, 0x01, 0xd0\n\t" + : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0)); + return xcr0_lo | ((xxh_u64)xcr0_hi << 32); +#endif +} +#endif + +#define XXH_SSE2_CPUID_MASK (1 << 26) +#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27)) +#define XXH_AVX2_CPUID_MASK (1 << 5) +#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1)) +#define XXH_AVX512F_CPUID_MASK (1 << 16) +#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1)) + +/*! + * @internal + * @brief Returns the best XXH3 implementation. + * + * Runs various CPUID/XGETBV tests to try and determine the best implementation. + * + * @ret The best @ref XXH_VECTOR implementation. + * @see XXH_VECTOR_TYPES + */ +static int XXH_featureTest(void) +{ + xxh_u32 abcd[4]; + xxh_u32 max_leaves; + int best = XXH_SCALAR; +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 + xxh_u64 xgetbv_val; +#endif +#if defined(__GNUC__) && defined(__i386__) + xxh_u32 cpuid_supported; + __asm__( + "# For the sake of ruthless backwards compatibility, check if CPUID\n\t" + "# is supported in the EFLAGS on i386.\n\t" + "# This is not necessary on x86_64 - CPUID is mandatory.\n\t" + "# The ID flag (bit 21) in the EFLAGS register indicates support\n\t" + "# for the CPUID instruction. If a software procedure can set and\n\t" + "# clear this flag, the processor executing the procedure supports\n\t" + "# the CPUID instruction.\n\t" + "# \n\t" + "#\n\t" + "# Routine is from .\n\t" + + "# Save EFLAGS\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# Store EFLAGS\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# Invert the ID bit in stored EFLAGS\n\t" + XXH_I_ATT("xor dword ptr[esp], 0x200000", "xorl $0x200000, (%%esp)") + "# Load stored EFLAGS (with ID bit inverted)\n\t" + XXH_I_ATT("popfd", "popfl" ) + "# Store EFLAGS again (ID bit may or not be inverted)\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t" + XXH_I_ATT("pop eax", "popl %%eax" ) + "# eax = whichever bits were changed\n\t" + XXH_I_ATT("xor eax, dword ptr[esp]", "xorl (%%esp), %%eax" ) + "# Restore original EFLAGS\n\t" + XXH_I_ATT("popfd", "popfl" ) + "# eax = zero if ID bit can't be changed, else non-zero\n\t" + XXH_I_ATT("and eax, 0x200000", "andl $0x200000, %%eax" ) + : "=a" (cpuid_supported) :: "cc"); + + if (XXH_unlikely(!cpuid_supported)) { + XXH_debugPrint("CPUID support is not detected!"); + return best; + } + +#endif + /* Check how many CPUID pages we have */ + XXH_cpuid(0, 0, abcd); + max_leaves = abcd[0]; + + /* Shouldn't happen on hardware, but happens on some QEMU configs. */ + if (XXH_unlikely(max_leaves == 0)) { + XXH_debugPrint("Max CPUID leaves == 0!"); + return best; + } + + /* Check for SSE2, OSXSAVE and xgetbv */ + XXH_cpuid(1, 0, abcd); + + /* + * Test for SSE2. The check is redundant on x86_64, but it doesn't hurt. + */ + if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK)) + return best; + + XXH_debugPrint("SSE2 support detected."); + + best = XXH_SSE2; +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 + /* Make sure we have enough leaves */ + if (XXH_unlikely(max_leaves < 7)) + return best; + + /* Test for OSXSAVE and XGETBV */ + if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK) + return best; + + /* CPUID check for AVX features */ + XXH_cpuid(7, 0, abcd); + + xgetbv_val = XXH_xgetbv(); +#if XXH_DISPATCH_AVX2 + /* Validate that AVX2 is supported by the CPU */ + if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK) + return best; + + /* Validate that the OS supports YMM registers */ + if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) { + XXH_debugPrint("AVX2 supported by the CPU, but not the OS."); + return best; + } + + /* AVX2 supported */ + XXH_debugPrint("AVX2 support detected."); + best = XXH_AVX2; +#endif +#if XXH_DISPATCH_AVX512 + /* Check if AVX512F is supported by the CPU */ + if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) { + XXH_debugPrint("AVX512F not supported by CPU"); + return best; + } + + /* Validate that the OS supports ZMM registers */ + if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) { + XXH_debugPrint("AVX512F supported by the CPU, but not the OS."); + return best; + } + + /* AVX512F supported */ + XXH_debugPrint("AVX512F support detected."); + best = XXH_AVX512; +#endif +#endif + return best; +} + + +/* === Vector implementations === */ + +/*! + * @internal + * @brief Defines the various dispatch functions. + * + * TODO: Consolidate? + * + * @param suffix The suffix for the functions, e.g. sse2 or scalar + * @param target XXH_TARGET_* or empty. + */ +#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \ + \ +/* === XXH3, default variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_default_##suffix(const void* XXH_RESTRICT input, size_t len) \ +{ \ + return XXH3_hashLong_64b_internal( \ + input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH3, Seeded variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \ + XXH64_hash_t seed) \ +{ \ + return XXH3_hashLong_64b_withSeed_internal( \ + input, len, seed, XXH3_accumulate_512_##suffix, \ + XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \ + ); \ +} \ + \ +/* === XXH3, Secret variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \ + const void* secret, size_t secretLen) \ +{ \ + return XXH3_hashLong_64b_internal( \ + input, len, secret, secretLen, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH3 update variants === */ \ + \ +XXH_NO_INLINE target XXH_errorcode \ +XXH3_update_##suffix(XXH3_state_t* state, const void* input, size_t len) \ +{ \ + return XXH3_update(state, (const xxh_u8*)input, len, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \ +} \ + \ +/* === XXH128 default variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_default_##suffix(const void* XXH_RESTRICT input, size_t len) \ +{ \ + return XXH3_hashLong_128b_internal( \ + input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH128 Secret variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \ + const void* XXH_RESTRICT secret, size_t secretLen) \ +{ \ + return XXH3_hashLong_128b_internal( \ + input, len, (const xxh_u8*)secret, secretLen, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \ +} \ + \ +/* === XXH128 Seeded variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \ + XXH64_hash_t seed) \ +{ \ + return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix, \ + XXH3_initCustomSecret_##suffix); \ +} + +/* End XXH_DEFINE_DISPATCH_FUNCS */ + +#if XXH_DISPATCH_SCALAR +XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */) +#endif +XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2) +#if XXH_DISPATCH_AVX2 +XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2) +#endif +#if XXH_DISPATCH_AVX512 +XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512) +#endif +#undef XXH_DEFINE_DISPATCH_FUNCS + +/* ==== Dispatchers ==== */ + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(const void* XXH_RESTRICT, size_t); + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t); + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t); + +typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH3_state_t*, const void*, size_t); + +typedef struct { + XXH3_dispatchx86_hashLong64_default hashLong64_default; + XXH3_dispatchx86_hashLong64_withSeed hashLong64_seed; + XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret; + XXH3_dispatchx86_update update; +} XXH_dispatchFunctions_s; + +#define XXH_NB_DISPATCHES 4 + +/*! + * @internal + * @brief Table of dispatchers for @ref XXH3_64bits(). + * + * @pre The indices must match @ref XXH_VECTOR_TYPE. + */ +static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = { +#if XXH_DISPATCH_SCALAR + /* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar }, +#else + /* Scalar */ { NULL, NULL, NULL, NULL }, +#endif + /* SSE2 */ { XXHL64_default_sse2, XXHL64_seed_sse2, XXHL64_secret_sse2, XXH3_update_sse2 }, +#if XXH_DISPATCH_AVX2 + /* AVX2 */ { XXHL64_default_avx2, XXHL64_seed_avx2, XXHL64_secret_avx2, XXH3_update_avx2 }, +#else + /* AVX2 */ { NULL, NULL, NULL, NULL }, +#endif +#if XXH_DISPATCH_AVX512 + /* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 } +#else + /* AVX512 */ { NULL, NULL, NULL, NULL } +#endif +}; +/*! + * @internal + * @brief The selected dispatch table for @ref XXH3_64bits(). + */ +static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL }; + + +typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(const void* XXH_RESTRICT, size_t); + +typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t); + +typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t); + +typedef struct { + XXH3_dispatchx86_hashLong128_default hashLong128_default; + XXH3_dispatchx86_hashLong128_withSeed hashLong128_seed; + XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret; + XXH3_dispatchx86_update update; +} XXH_dispatch128Functions_s; + + +/*! + * @internal + * @brief Table of dispatchers for @ref XXH3_128bits(). + * + * @pre The indices must match @ref XXH_VECTOR_TYPE. + */ +static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = { +#if XXH_DISPATCH_SCALAR + /* Scalar */ { XXHL128_default_scalar, XXHL128_seed_scalar, XXHL128_secret_scalar, XXH3_update_scalar }, +#else + /* Scalar */ { NULL, NULL, NULL, NULL }, +#endif + /* SSE2 */ { XXHL128_default_sse2, XXHL128_seed_sse2, XXHL128_secret_sse2, XXH3_update_sse2 }, +#if XXH_DISPATCH_AVX2 + /* AVX2 */ { XXHL128_default_avx2, XXHL128_seed_avx2, XXHL128_secret_avx2, XXH3_update_avx2 }, +#else + /* AVX2 */ { NULL, NULL, NULL, NULL }, +#endif +#if XXH_DISPATCH_AVX512 + /* AVX512 */ { XXHL128_default_avx512, XXHL128_seed_avx512, XXHL128_secret_avx512, XXH3_update_avx512 } +#else + /* AVX512 */ { NULL, NULL, NULL, NULL } +#endif +}; + +/*! + * @internal + * @brief The selected dispatch table for @ref XXH3_64bits(). + */ +static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL }; + +/*! + * @internal + * @brief Runs a CPUID check and sets the correct dispatch tables. + */ +static void XXH_setDispatch(void) +{ + int vecID = XXH_featureTest(); + XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1); + assert(XXH_SCALAR <= vecID && vecID <= XXH_AVX512); +#if !XXH_DISPATCH_SCALAR + assert(vecID != XXH_SCALAR); +#endif +#if !XXH_DISPATCH_AVX512 + assert(vecID != XXH_AVX512); +#endif +#if !XXH_DISPATCH_AVX2 + assert(vecID != XXH_AVX2); +#endif + XXH_g_dispatch = XXH_kDispatch[vecID]; + XXH_g_dispatch128 = XXH_kDispatch128[vecID]; +} + + +/* ==== XXH3 public functions ==== */ + +static XXH64_hash_t +XXH3_hashLong_64b_defaultSecret_selection(const void* input, size_t len, + XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + if (XXH_g_dispatch.hashLong64_default == NULL) XXH_setDispatch(); + return XXH_g_dispatch.hashLong64_default(input, len); +} + +XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_defaultSecret_selection); +} + +static XXH64_hash_t +XXH3_hashLong_64b_withSeed_selection(const void* input, size_t len, + XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + if (XXH_g_dispatch.hashLong64_seed == NULL) XXH_setDispatch(); + return XXH_g_dispatch.hashLong64_seed(input, len, seed64); +} + +XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed_selection); +} + +static XXH64_hash_t +XXH3_hashLong_64b_withSecret_selection(const void* input, size_t len, + XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen) +{ + (void)seed64; + if (XXH_g_dispatch.hashLong64_secret == NULL) XXH_setDispatch(); + return XXH_g_dispatch.hashLong64_secret(input, len, secret, secretLen); +} + +XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_64b_withSecret_selection); +} + +XXH_errorcode +XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len) +{ + if (XXH_g_dispatch.update == NULL) XXH_setDispatch(); + return XXH_g_dispatch.update(state, (const xxh_u8*)input, len); +} + + +/* ==== XXH128 public functions ==== */ + +static XXH128_hash_t +XXH3_hashLong_128b_defaultSecret_selection(const void* input, size_t len, + XXH64_hash_t seed64, const void* secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + if (XXH_g_dispatch128.hashLong128_default == NULL) XXH_setDispatch(); + return XXH_g_dispatch128.hashLong128_default(input, len); +} + +XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_defaultSecret_selection); +} + +static XXH128_hash_t +XXH3_hashLong_128b_withSeed_selection(const void* input, size_t len, + XXH64_hash_t seed64, const void* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + if (XXH_g_dispatch128.hashLong128_seed == NULL) XXH_setDispatch(); + return XXH_g_dispatch128.hashLong128_seed(input, len, seed64); +} + +XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_128b_withSeed_selection); +} + +static XXH128_hash_t +XXH3_hashLong_128b_withSecret_selection(const void* input, size_t len, + XXH64_hash_t seed64, const void* secret, size_t secretLen) +{ + (void)seed64; + if (XXH_g_dispatch128.hashLong128_secret == NULL) XXH_setDispatch(); + return XXH_g_dispatch128.hashLong128_secret(input, len, secret, secretLen); +} + +XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen) +{ + return XXH3_128bits_internal(input, len, 0, secret, secretLen, XXH3_hashLong_128b_withSecret_selection); +} + +XXH_errorcode +XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len) +{ + if (XXH_g_dispatch128.update == NULL) XXH_setDispatch(); + return XXH_g_dispatch128.update(state, (const xxh_u8*)input, len); +} + +#if defined (__cplusplus) +} +#endif +/*! @} */ diff --git a/External/xxHash/0.8.1/xxh_x86dispatch.h b/External/xxHash/0.8.1/xxh_x86dispatch.h new file mode 100644 index 000000000..6bc17bcbb --- /dev/null +++ b/External/xxHash/0.8.1/xxh_x86dispatch.h @@ -0,0 +1,86 @@ +/* + * xxHash - XXH3 Dispatcher for x86-based targets + * Copyright (C) 2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#ifndef XXH_X86DISPATCH_H_13563687684 +#define XXH_X86DISPATCH_H_13563687684 + +#include "xxhash.h" /* XXH64_hash_t, XXH3_state_t */ + +#if defined (__cplusplus) +extern "C" { +#endif + +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_dispatch(const void* input, size_t len); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen); +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len); + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_dispatch(const void* input, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed_dispatch(const void* input, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret_dispatch(const void* input, size_t len, const void* secret, size_t secretLen); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH3_state_t* state, const void* input, size_t len); + +#if defined (__cplusplus) +} +#endif + + +/* automatic replacement of XXH3 functions. + * can be disabled by setting XXH_DISPATCH_DISABLE_REPLACE */ +#ifndef XXH_DISPATCH_DISABLE_REPLACE + +# undef XXH3_64bits +# define XXH3_64bits XXH3_64bits_dispatch +# undef XXH3_64bits_withSeed +# define XXH3_64bits_withSeed XXH3_64bits_withSeed_dispatch +# undef XXH3_64bits_withSecret +# define XXH3_64bits_withSecret XXH3_64bits_withSecret_dispatch +# undef XXH3_64bits_update +# define XXH3_64bits_update XXH3_64bits_update_dispatch + +# undef XXH128 +# define XXH128 XXH3_128bits_withSeed_dispatch +# define XXH3_128bits XXH3_128bits_dispatch +# undef XXH3_128bits +# define XXH3_128bits XXH3_128bits_dispatch +# undef XXH3_128bits_withSeed +# define XXH3_128bits_withSeed XXH3_128bits_withSeed_dispatch +# undef XXH3_128bits_withSecret +# define XXH3_128bits_withSecret XXH3_128bits_withSecret_dispatch +# undef XXH3_128bits_update +# define XXH3_128bits_update XXH3_128bits_update_dispatch + +#endif /* XXH_DISPATCH_DISABLE_REPLACE */ + +#endif /* XXH_X86DISPATCH_H_13563687684 */ diff --git a/External/xxHash/0.8.1/xxhash.c b/External/xxHash/0.8.1/xxhash.c new file mode 100644 index 000000000..0fae88c5d --- /dev/null +++ b/External/xxHash/0.8.1/xxhash.c @@ -0,0 +1,43 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/External/xxHash/0.8.1/xxhash.h b/External/xxHash/0.8.1/xxhash.h new file mode 100644 index 000000000..08ab79457 --- /dev/null +++ b/External/xxHash/0.8.1/xxhash.h @@ -0,0 +1,5580 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (C) 2012-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +/* TODO: update */ +/* Notice extracted from xxHash homepage: + +xxHash is an extremely fast hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MurmurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +Note: SMHasher's CRC32 implementation is not the fastest one. +Other speed-oriented implementations can be faster, +especially in combination with PCLMUL instruction: +https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + + + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 1 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return `XXH_VERSION_NUMBER` of the invoked library. + */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# else +# if ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +# endif +#endif + +/*! + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that @ref xxh3_family provides competitive speed + * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif + +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* +Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute +introduced in CPP17 and C23. +CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough +C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough +*/ +#if XXH_HAS_C_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_CPP_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((fallthrough)) +#else +# define XXH_FALLTHROUGH +#endif + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. + * + * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/* XXH3_64bits(): + * default 64-bit variant, using default secret and default seed of 0. + * It's the fastest variant. */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); + +/* + * XXH3_64bits_withSeed(): + * This variant generates a custom secret on the fly + * based on default secret altered using the `seed` value. + * While this operation is decently fast, note that it's not completely free. + * Note: seed==0 produces the same results as XXH3_64bits(). + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing "XXH3_generateSecret()" instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); + +/* + * XXH3_64bits_reset(): + * Initialize with default parameters. + * digest will be equivalent to `XXH3_64bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); +/* + * XXH3_64bits_reset_withSeed(): + * Generate a custom secret from `seed`, and store it into `statePtr`. + * digest will be equivalent to `XXH3_64bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +/* + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * XXH128_cmp(): + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); + + +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read or write to it, it may be removed. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it, it may be removed. */ +}; /* typedef'd to XXH64_state_t */ + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; } + + +/* XXH128() : + * simple alias to pre-selected XXH3_128bits variant + */ +XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/* + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, + * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @secretSize + * into an already allocated buffer @secretBuffer. + * @secretSize must be >= XXH3_SECRET_SIZE_MIN + * + * The generated secret can then be used with any `*_withSecret()` variant. + * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, + * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so + * XXH3_generateSecret() can be employed to ensure proper quality. + * + * customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes. + * The resulting `secret` will nonetheless provide all required qualities. + * + * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize); + + +/* + * XXH3_generateSecret_fromSeed(): + * + * Generate the same secret as the _withSeed() variants. + * + * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily). + * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * This generator is notably useful in combination with `_withSecretandSeed()`, + * as a way to emulate a faster `_withSeed()` variant. + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed); + +/* + * *_withSecretandSeed() : + * These variants generate hash values using either + * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes) + * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, + * via its impact to the seed. + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed); + +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + + +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64 & arm64, + * which are platforms known to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ +# if !defined(__clang__) && \ +( \ + (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + ( \ + defined(__GNUC__) && ( \ + (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ + ( \ + defined(__mips__) && \ + (__mips <= 5 || __mips_isa_rev < 6) && \ + (!defined(__mips16) || defined(__mips_mips16e2)) \ + ) \ + ) \ + ) \ +) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ + || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# define XXH_ASSERT(c) ((void)0) +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# include +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; + return ((const xxh_unalign*)ptr)->u32; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 h32) +{ + h32 ^= h32 >> 15; + h32 *= XXH_PRIME32_2; + h32 ^= h32 >> 13; + h32 *= XXH_PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ +static xxh_u32 +XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + h32 += (*ptr++) * XXH_PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(h32); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 4: XXH_PROCESS4; + return XXH32_avalanche(h32); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 0: return XXH32_avalanche(h32); + } + XXH_ASSERT(0); + return h32; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +/*! + * @ingroup xxh32_family + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)); + state.v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + state.v[1] = seed + XXH_PRIME32_2; + state.v[2] = seed + 0; + state.v[3] = seed - XXH_PRIME32_1; + /* do not write into reserved, planned to be removed in a future version */ + XXH_memcpy(statePtr, &state, sizeof(state) - sizeof(state.reserved)); + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! + * @ingroup xxh32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer, but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; + return ((const xxh_unalign64*)ptr)->u64; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +static xxh_u64 XXH64_avalanche(xxh_u64 h64) +{ + h64 ^= h64 >> 33; + h64 *= XXH_PRIME64_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +static xxh_u64 +XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(h64); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (inputtotal_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#else +/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#if defined(__GNUC__) +# if defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# elif defined(__ARM_NEON__) || defined(__ARM_NEON) +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# endif +#elif defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment reqired for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM64) || defined(_M_ARM_ARMV7VE) /* msvc */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + + +#if XXH_VECTOR == XXH_NEON +/* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split into + * two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] + * | '---------. .--------' | + * | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 is + * completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so modifying + * D11 will modify the high half of Q5. This is similar to how modifying AH + * will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd lanes + * in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead of + * taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the + * lower and upper halves of the Q register to end up with the high and low + * halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions modifying two + * registers at once is not possible in GCC or Clang's IR, and they have to + * create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, many + * quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes to a + * D-form register zero the high bits, similar to how writes to W-form scalar + * registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent + * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one + * operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be this + * mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } + * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } + * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single VZIP.32. + */ + +/*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) + * { + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ +# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && defined(__GNUC__) \ + && !defined(__aarch64__) && !defined(__arm64__) && !defined(_M_ARM64) +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ + __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ + (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + } while (0) +# else +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + (outLo) = vmovn_u64 (in); \ + (outHi) = vshrn_n_u64 ((in), 32); \ + } while (0) +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +# if defined(__s390x__) +# include +# else +/* gcc's altivec.h can have the unwanted consequence to unconditionally + * #define bool, vector, and pixel keywords, + * with bad consequences for programs already using these keywords for other purposes. + * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, + * but it seems that, in some cases, it isn't. + * Force the build macro to be defined, so that keywords are not altered. + */ +# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) +# define __APPLE_ALTIVEC__ +# endif +# include +# endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# include +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if defined(__GNUC__) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif defined(_M_X64) || defined(_M_IA64) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= 0x165667919E3779F9ULL; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= 0x9FB21C651E98DF25ULL; + h64 ^= (h64 >> 35) + len ; + h64 *= 0x9FB21C651E98DF25ULL; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); + + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + int const nbRounds = (int)len / 16; + int i; + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + acc = XXH3_avalanche(acc); + XXH_ASSERT(nbRounds >= 8); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + /* last bytes */ + acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + return XXH3_avalanche(acc); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', + * this will warn "discards 'const' qualifier". */ + union { + const __m512i* cp; + void* p; + } remote_const_void; + remote_const_void.cp = src + i; + dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { + uint64x2_t* const xacc = (uint64x2_t *) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* const xinput = (const uint8_t *) input; + uint8_t const* const xsecret = (const uint8_t *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN / sizeof(uint64x2_t); i++) { + /* data_vec = xinput[i]; */ + uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key; + uint32x2_t data_key_lo, data_key_hi; + /* xacc[i] += swap(data_vec); */ + uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); + uint64x2_t const swapped = vextq_u64(data64, data64, 1); + xacc[i] = vaddq_u64 (xacc[i], swapped); + /* data_key = data_vec ^ key_vec; */ + data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); + /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (data_key >> 32); + * data_key = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ + xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); + + } + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { uint64x2_t* xacc = (uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(uint64x2_t); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); + uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec)); + + /* xacc[i] *= XXH_PRIME32_1 */ + uint32x2_t data_key_lo, data_key_hi; + /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (xacc[i] >> 32); + * xacc[i] = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + { /* + * prod_hi = (data_key >> 32) * XXH_PRIME32_1; + * + * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will + * incorrectly "optimize" this: + * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); + * shifted = vshll_n_u32(tmp, 32); + * to this: + * tmp = "vmulq_u64"(a, b); // no such thing! + * shifted = vshlq_n_u64(tmp, 32); + * + * However, unlike SSE, Clang lacks a 64-bit multiply routine + * for NEON, and it scalarizes two 64-bit multiplies instead. + * + * vmull_u32 has the same timing as vmul_u32, and it avoids + * this bug completely. + * See https://bugs.llvm.org/show_bug.cgi?id=39967 + */ + uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); + /* xacc[i] = prod_hi << 32; */ + xacc[i] = vshlq_n_u64(prod_hi, 32); + /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); + } + } } +} + +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + unsigned long long* const xacc = (unsigned long long*) acc; + xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ + xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = vec_xl(0, xacc + 2 * i); + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + /* xacc[i] = acc_vec; */ + vec_xst(acc_vec, 0, xacc + 2 * i); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_u64x2* const xacc = (xxh_u64x2*) acc; + const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +/* scalar variants - universal */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xinput = (const xxh_u8*) input; /* no alignment restriction */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + size_t i; + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + for (i=0; i < XXH_ACC_NB; i++) { + xxh_u64 const data_val = XXH_readLE64(xinput + 8*i); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + i*8); + xacc[i ^ 1] += data_val; /* swap adjacent lanes */ + xacc[i] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + size_t i; + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + for (i=0; i < XXH_ACC_NB; i++) { + xxh_u64 const key64 = XXH_readLE64(xsecret + 8*i); + xxh_u64 acc64 = xacc[i]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[i] = acc64; + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__clang__) && defined(__aarch64__) + /* + * UGLY HACK: + * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), long MOVK chains stall the + * integer pipelines: + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes Clang to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + /* + * Note: in debug mode, this overrides the asm optimization + * and Clang will emit MOVK chains again. + */ + XXH_ASSERT(kSecretPtr == XXH3_kSecret); + + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes Clang to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + + + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * XXH3_accumulate() + * Loops over XXH3_accumulate_512(). + * Assumption: nbStripes will not overflow the secret size + */ +XXH_FORCE_INLINE void +XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes, + XXH3_f_accumulate_512 f_acc512) +{ + size_t n; + for (n = 0; n < nbStripes; n++ ) { + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; + XXH_PREFETCH(in + XXH_PREFETCH_DIST); + f_acc512(acc, + in, + secret + n*XXH_SECRET_CONSUME_RATE); + } +} + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* input, size_t len, + XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ + +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/* Note : when XXH3_consumeStripes() is invoked, + * there must be a guarantee that at least one more byte must be consumed from input + * so that the function can blindly consume all stripes using the "normal" secret segment */ +XXH_FORCE_INLINE void +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ + XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); + if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { + /* need a scrambling operation */ + size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; + size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); + f_scramble(acc, secret + secretLimit); + XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); + *nbStripesSoFarPtr = nbStripesAfterBlock; + } else { + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); + *nbStripesSoFarPtr += nbStripes; + } +} + +#ifndef XXH3_STREAM_USE_STACK +# ifndef __clang__ /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + + /* large input to consume : ingest per full block */ + if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar); + /* join to current block's end */ + { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar; + XXH_ASSERT(nbStripes <= nbStripes); + XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512); + f_scramble(acc, secret + state->secretLimit); + state->nbStripesSoFar = 0; + input += nbStripesToEnd * XXH_STRIPE_LEN; + nbStripes -= nbStripesToEnd; + } + /* consume per entire blocks */ + while(nbStripes >= state->nbStripesPerBlock) { + XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + state->secretLimit); + input += state->nbStripesPerBlock * XXH_STRIPE_LEN; + nbStripes -= state->nbStripesPerBlock; + } + /* consume last partial block */ + XXH3_accumulate(acc, input, secret, nbStripes, f_acc512); + input += nbStripes * XXH_STRIPE_LEN; + XXH_ASSERT(input < bEnd); /* at least some bytes left */ + state->nbStripesSoFar = nbStripes; + /* buffer predecessor of last partial stripe */ + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN); + } else { + /* content to consume <= block size */ + /* Consume input by a multiple of internal buffer size */ + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; + do { + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + input += XXH3_INTERNALBUFFER_SIZE; + } while (inputbuffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + } + } + + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate_512, XXH3_scrambleAcc); + /* last stripe */ + XXH3_accumulate_512(acc, + state->buffer + state->bufferedSize - XXH_STRIPE_LEN, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } else { /* bufferedSize < XXH_STRIPE_LEN */ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + XXH3_accumulate_512(acc, + lastStripe, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + int const nbRounds = (int)len / 32; + int i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + for (i=0; i<4; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + (32 * i), + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + XXH_ASSERT(nbRounds >= 4); + for (i=4 ; i < nbRounds; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + 0ULL - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ + +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +static void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize) +{ + XXH_ASSERT(secretBuffer != NULL); + if (secretBuffer == NULL) return XXH_ERROR; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } + XXH_ASSERT(customSeed != NULL); + if (customSeed == NULL) return XXH_ERROR; + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n-config [path] + -config <path> Explicitly specify the config file to use. @@ -331,13 +331,13 @@

FBuild.exe Detailed

Instead of building specified targets generate a JSON compilation database for them. Resulting compilation database will include entries for all source files from ObjectList and Library nodes that are dependencies of the specified targets.

-
-config [path]
+
-config <path>

Explicitly specify the config file to use. By default, FASTBuild looks for "fbuild.bff" in the current directory. This options allows a file to be explicitly specified instead.

-
--continueafterdbmove
+
-continueafterdbmove

Allow build to continue after a DB move.

FASTBuild's database is tied to the directory in which it was created and cannot be moved. If a move is detected, an error will be emitted. -continueafterdbmove allows the build From f131b2dc6b1b9607617de99b558435a9be258466 Mon Sep 17 00:00:00 2001 From: Troels Gram <28827711+to01z@users.noreply.github.com> Date: Sat, 17 Jun 2023 18:26:53 -0700 Subject: [PATCH 040/129] Add -dbfile option to set dependency database file explicitly (#970) * Add support for -dbfile option to set dependency graph file explicitly * Add DBLocation test to TestGraph suite * Fix test database deletion * Added docs to -dbfile option --- .../FBuild/Documentation/docs/options.html | 12 +++++- Code/Tools/FBuild/FBuildCore/FBuild.cpp | 28 ++++++++----- .../Tools/FBuild/FBuildCore/FBuildOptions.cpp | 20 ++++++++++ Code/Tools/FBuild/FBuildCore/FBuildOptions.h | 1 + .../TestGraph/DatabaseLocation/fbuild.bff | 16 ++++++++ .../FBuild/FBuildTest/Tests/FBuildTest.cpp | 7 ++++ .../FBuild/FBuildTest/Tests/FBuildTest.h | 2 + .../FBuild/FBuildTest/Tests/TestGraph.cpp | 40 +++++++++++++++++++ 8 files changed, 115 insertions(+), 11 deletions(-) create mode 100644 Code/Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff diff --git a/Code/Tools/FBuild/Documentation/docs/options.html b/Code/Tools/FBuild/Documentation/docs/options.html index a7fed61bb..332c06eae 100644 --- a/Code/Tools/FBuild/Documentation/docs/options.html +++ b/Code/Tools/FBuild/Documentation/docs/options.html @@ -71,6 +71,10 @@

FBuild.exe

-continueafterdbmove Allow build to continue after a DB move. + + -dbfile <path> + Explicitly specify the dependency database file to use. + -debug [Windows Only] Allow attaching a debugger immediately on startup. @@ -344,7 +348,13 @@

FBuild.exe Detailed

to continue after this error has been emitted, ignoring and replacing the DB file.

-
-debug
+
-dbfile <path>
+
+

Explicitly specify the dependency database file to use. By default, FASTBuild will load and save its dependency database in the same directory as the config file +(with a ".platform.fdb" suffix). This option allows the file to be explicitly specified instead.

+
+ +
-debug

[Windows Only] Display a message box on startup to allow a debugger to be attached. Can be useful if triaging problems with FASTBuild that can't be reproduced in the debugger.

diff --git a/Code/Tools/FBuild/FBuildCore/FBuild.cpp b/Code/Tools/FBuild/FBuildCore/FBuild.cpp index fa3b41aac..0ce5a1002 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuild.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuild.cpp @@ -154,18 +154,26 @@ bool FBuild::Initialize( const char * nodeGraphDBFile ) } else { - m_DependencyGraphFile = bffFile; - if ( m_DependencyGraphFile.EndsWithI( ".bff" ) ) + if ( m_Options.m_DBFile.IsEmpty() ) { - m_DependencyGraphFile.SetLength( m_DependencyGraphFile.GetLength() - 4 ); + m_DependencyGraphFile = bffFile; + if ( m_DependencyGraphFile.EndsWithI( ".bff" ) ) + { + m_DependencyGraphFile.SetLength( m_DependencyGraphFile.GetLength() - 4 ); + } + #if defined( __WINDOWS__ ) + m_DependencyGraphFile += ".windows.fdb"; + #elif defined( __OSX__ ) + m_DependencyGraphFile += ".osx.fdb"; + #elif defined( __LINUX__ ) + m_DependencyGraphFile += ".linux.fdb"; + #endif + } + else + { + // DB filename explicitly set on command line + m_DependencyGraphFile = m_Options.m_DBFile; } - #if defined( __WINDOWS__ ) - m_DependencyGraphFile += ".windows.fdb"; - #elif defined( __OSX__ ) - m_DependencyGraphFile += ".osx.fdb"; - #elif defined( __LINUX__ ) - m_DependencyGraphFile += ".linux.fdb"; - #endif } SmallBlockAllocator::SetSingleThreadedMode( true ); diff --git a/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp b/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp index 437e5a816..1491d66d4 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp @@ -222,6 +222,25 @@ FBuildOptions::OptionsResult FBuildOptions::ProcessCommandLine( int argc, char * m_Args += '"'; continue; } + else if ( thisArg == "-dbfile" ) + { + const int32_t pathIndex = ( i + 1 ); + if ( pathIndex >= argc ) + { + OUTPUT( "FBuild: Error: Missing for '-dbfile' argument\n" ); + OUTPUT( "Try \"%s -help\"\n", programName.Get() ); + return OPTIONS_ERROR; + } + m_DBFile = argv[ pathIndex ]; + i++; // skip extra arg we've consumed + + // add to args we might pass to subprocess + m_Args += ' '; + m_Args += '"'; // surround db file with quotes to avoid problems with spaces in the path + m_Args += m_DBFile; + m_Args += '"'; + continue; + } #if defined( __WINDOWS__ ) else if ( thisArg == "-debug" ) { @@ -624,6 +643,7 @@ void FBuildOptions::DisplayHelp( const AString & programName ) const " -config Explicitly specify the config file to use.\n" " -continueafterdbmove\n" " Allow builds after a DB move.\n" + " -dbfile Explicitly specify the dependency database file to use.\n" " -debug (Windows) Break at startup, to attach debugger.\n" " -dist Allow distributed compilation.\n" " -distverbose Print detailed info for distributed compilation.\n" diff --git a/Code/Tools/FBuild/FBuildCore/FBuildOptions.h b/Code/Tools/FBuild/FBuildCore/FBuildOptions.h index a37086833..1a107fb37 100755 --- a/Code/Tools/FBuild/FBuildCore/FBuildOptions.h +++ b/Code/Tools/FBuild/FBuildCore/FBuildOptions.h @@ -99,6 +99,7 @@ class FBuildOptions bool m_FixupErrorPaths = false; bool m_ForceDBMigration_Debug = false; // Force migration even if bff has not changed (for tests) bool m_ContinueAfterDBMove = false; + AString m_DBFile; uint32_t m_NumWorkerThreads = 0; // True default detected in constructor AString m_ConfigFile; diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff new file mode 100644 index 000000000..0f7b91298 --- /dev/null +++ b/Code/Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff @@ -0,0 +1,16 @@ +// +// DatabaseLocation +// +//------------------------------------------------------------------------------ + +// Use the standard test environment +//------------------------------------------------------------------------------ +#include "../../testcommon.bff" +Using( .StandardEnvironment ) +Settings {} + +TextFile( 'TestTarget' ) +{ + .TextFileOutput = '$Out$/Test/Graph/DatabaseLocation/test.txt' + .TextFileInputStrings = { 'Test content' } +} diff --git a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp index 8d092de79..392a41d06 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp @@ -345,6 +345,13 @@ void FBuildForTest::SerializeDepGraphToText( const char * nodeName, AString & ou m_DependencyGraph->SerializeToText( deps, outBuffer ); } +// GetDependencyGraphFile +//------------------------------------------------------------------------------ +const char * FBuildForTest::GetDependencyGraphFile() const +{ + return m_DependencyGraphFile.Get(); +} + // Build //------------------------------------------------------------------------------ /*virtual*/ bool FBuildForTest::Build( Node * nodeToBuild ) diff --git a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h index f5acea659..527c0e4d5 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h +++ b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h @@ -94,6 +94,8 @@ class FBuildForTest : public FBuild void SerializeDepGraphToText( const char * nodeName, AString & outBuffer ) const; + const char * GetDependencyGraphFile() const; + using FBuild::Build; virtual bool Build( Node * nodeToBuild ) override; }; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp index 8fba02ce2..f40afcba6 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp @@ -57,6 +57,7 @@ class TestGraph : public FBuildTest void DBVersionChanged() const; void FixupErrorPaths() const; void CyclicDependency() const; + void DBLocation() const; }; // Register Tests @@ -78,6 +79,7 @@ REGISTER_TESTS_BEGIN( TestGraph ) REGISTER_TEST( DBVersionChanged ) REGISTER_TEST( FixupErrorPaths ) REGISTER_TEST( CyclicDependency ) + REGISTER_TEST( DBLocation ) REGISTER_TESTS_END // NodeTestHelper @@ -1005,4 +1007,42 @@ void TestGraph::CyclicDependency() const } } +// DBLocation +//------------------------------------------------------------------------------ +void TestGraph::DBLocation() const +{ + FBuildTestOptions options; + options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff"; + options.m_SaveDBOnCompletion = true; + + AString dbFileDefaultLocation( "Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/" ); + AString dbFileExplicitLocation( "../tmp/Test/Graph/DatabaseLocation/GraphDB.fdb" ); + + // Build a target and let serialization save to default location + { + FBuildForTest fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); + + AString dbFile( fBuild.GetDependencyGraphFile() ); + EnsureFileDoesNotExist( dbFile ); + + TEST_ASSERT( fBuild.Build( "TestTarget" ) ); + TEST_ASSERT( PathUtils::PathBeginsWith( dbFile, dbFileDefaultLocation ) ); + } + + // Build a target and let serialization save to explicitly specified location + { + options.m_DBFile = dbFileExplicitLocation; + + FBuildForTest fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); + + AString dbFile( fBuild.GetDependencyGraphFile() ); + EnsureFileDoesNotExist( dbFile ); + + TEST_ASSERT( fBuild.Build( "TestTarget" ) ); + TEST_ASSERT( PathUtils::ArePathsEqual( dbFile, dbFileExplicitLocation ) ); + } +} + //------------------------------------------------------------------------------ From 1daa5005523ee5cd6c17f62d391368311ccf23bb Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 18 Jun 2023 11:09:33 +0930 Subject: [PATCH 041/129] Post-integration: -dbfile cleanup - avoid an unnecessary string copy --- Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp | 7 ------- Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h | 2 +- Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp | 4 ++-- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp index 392a41d06..8d092de79 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp @@ -345,13 +345,6 @@ void FBuildForTest::SerializeDepGraphToText( const char * nodeName, AString & ou m_DependencyGraph->SerializeToText( deps, outBuffer ); } -// GetDependencyGraphFile -//------------------------------------------------------------------------------ -const char * FBuildForTest::GetDependencyGraphFile() const -{ - return m_DependencyGraphFile.Get(); -} - // Build //------------------------------------------------------------------------------ /*virtual*/ bool FBuildForTest::Build( Node * nodeToBuild ) diff --git a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h index 527c0e4d5..fcfbcb296 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h +++ b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h @@ -94,7 +94,7 @@ class FBuildForTest : public FBuild void SerializeDepGraphToText( const char * nodeName, AString & outBuffer ) const; - const char * GetDependencyGraphFile() const; + const AString & GetDependencyGraphFile() const { return m_DependencyGraphFile; } using FBuild::Build; virtual bool Build( Node * nodeToBuild ) override; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp index f40afcba6..28cadff92 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp @@ -1023,7 +1023,7 @@ void TestGraph::DBLocation() const FBuildForTest fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); - AString dbFile( fBuild.GetDependencyGraphFile() ); + const AString & dbFile( fBuild.GetDependencyGraphFile() ); EnsureFileDoesNotExist( dbFile ); TEST_ASSERT( fBuild.Build( "TestTarget" ) ); @@ -1037,7 +1037,7 @@ void TestGraph::DBLocation() const FBuildForTest fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); - AString dbFile( fBuild.GetDependencyGraphFile() ); + const AString & dbFile( fBuild.GetDependencyGraphFile() ); EnsureFileDoesNotExist( dbFile ); TEST_ASSERT( fBuild.Build( "TestTarget" ) ); From b0e36d42cad1cb66edda78f9c454927f2edf3e66 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 2 Jul 2023 15:47:14 +0930 Subject: [PATCH 042/129] Cleanup node creation logic: - eliminate most bespoke functions and mostly consolidate around a single CreateNode function --- .../FBuildCore/BFF/Functions/Function.cpp | 8 +- .../FBuildCore/BFF/Functions/FunctionCopy.cpp | 2 +- .../BFF/Functions/FunctionRemoveDir.cpp | 2 +- .../BFF/Functions/FunctionSettings.cpp | 2 +- .../FBuild/FBuildCore/Graph/CopyDirNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/Node.cpp | 42 +-- Code/Tools/FBuild/FBuildCore/Graph/Node.h | 1 - .../FBuild/FBuildCore/Graph/NodeGraph.cpp | 329 +++--------------- .../Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 31 +- .../FBuildCore/Graph/ObjectListNode.cpp | 2 +- .../Tools/FBuild/FBuildTest/Tests/TestExe.cpp | 2 +- .../FBuild/FBuildTest/Tests/TestGraph.cpp | 139 ++------ .../FBuild/FBuildTest/Tests/TestRemoveDir.cpp | 16 - .../FBuild/FBuildTest/Tests/TestTest.cpp | 18 - 14 files changed, 99 insertions(+), 497 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp index c0951e8ee..e4c0c3a3e 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp @@ -562,8 +562,8 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, Node * node = nodeGraph.FindNode( name ); if ( node == nullptr ) { - node = nodeGraph.CreateDirectoryListNode( name ); - DirectoryListNode * dln = node->CastTo< DirectoryListNode >(); + DirectoryListNode * dln = nodeGraph.CreateNode( name ); + node = dln; dln->m_Path = path; if ( patterns ) { @@ -645,7 +645,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, } // Implicitly create the new node - compilerNode = nodeGraph.CreateCompilerNode( nodeName ); + compilerNode = nodeGraph.CreateNode( nodeName ); VERIFY( compilerNode->GetReflectionInfoV()->SetProperty( compilerNode, "Executable", compiler ) ); VERIFY( compilerNode->GetReflectionInfoV()->SetProperty( compilerNode, "AllowDistribution", false ) ); const char * lastSlash = compiler.FindLast( NATIVE_SLASH ); @@ -929,7 +929,7 @@ bool Function::ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Depen } // create an alias against the node - AliasNode * an = nodeGraph.CreateAliasNode( m_AliasForFunction ); + AliasNode * an = nodeGraph.CreateNode( m_AliasForFunction ); an->m_StaticDependencies = nodesToAlias; // TODO: make this use m_Targets & Initialize() // clear the string so it can't be used again diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp index 4a62b419e..102f8aca1 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp @@ -155,7 +155,7 @@ FunctionCopy::FunctionCopy() } // create our node - CopyFileNode * copyFileNode = nodeGraph.CreateCopyFileNode( dst ); + CopyFileNode * copyFileNode = nodeGraph.CreateNode( dst ); copyFileNode->m_Source = srcNode->GetName(); copyFileNode->m_PreBuildDependencyNames = preBuildDependencyNames; if ( !copyFileNode->Initialize( nodeGraph, funcStartIter, this ) ) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp index 7a4aacfc6..30fa7e982 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp @@ -42,7 +42,7 @@ FunctionRemoveDir::FunctionRemoveDir() return false; } - RemoveDirNode * removeDirNode = nodeGraph.CreateRemoveDirNode( m_AliasForFunction ); + Node * removeDirNode = nodeGraph.CreateNode( m_AliasForFunction ); if ( !PopulateProperties( nodeGraph, funcStartIter, removeDirNode ) ) { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp index f8ba7309e..5e7084262 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp @@ -34,7 +34,7 @@ FunctionSettings::FunctionSettings() return false; } - SettingsNode * settingsNode = nodeGraph.CreateSettingsNode( name ); + SettingsNode * settingsNode = nodeGraph.CreateNode( name ); if ( !PopulateProperties( nodeGraph, funcStartIter, settingsNode ) ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp index ae63bf32b..3541c1fd5 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp @@ -135,7 +135,7 @@ CopyDirNode::~CopyDirNode() = default; Node * n = nodeGraph.FindNode( dstFile ); if ( n == nullptr ) { - CopyFileNode * copyFileNode = nodeGraph.CreateCopyFileNode( dstFile ); + CopyFileNode * copyFileNode = nodeGraph.CreateNode( dstFile ); copyFileNode->m_Source = srcFileNode->GetName(); copyFileNode->m_PreBuildDependencyNames = preBuildDependencyNames; // inherit PreBuildDependencies const BFFToken * token = nullptr; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp index 8586bec10..92fe09d31 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp @@ -327,46 +327,6 @@ void Node::SetLastBuildTime( uint32_t ms ) AtomicStoreRelaxed( &m_LastBuildTimeMs, ms ); } -// CreateNode -//------------------------------------------------------------------------------ -/*static*/ Node * Node::CreateNode( NodeGraph & nodeGraph, Node::Type nodeType, const AString & name ) -{ - switch ( nodeType ) - { - case Node::PROXY_NODE: ASSERT( false ); return nullptr; - case Node::COPY_FILE_NODE: return nodeGraph.CreateCopyFileNode( name ); - case Node::DIRECTORY_LIST_NODE: return nodeGraph.CreateDirectoryListNode( name ); - case Node::EXEC_NODE: return nodeGraph.CreateExecNode( name ); - case Node::FILE_NODE: return nodeGraph.CreateFileNode( name ); - case Node::LIBRARY_NODE: return nodeGraph.CreateLibraryNode( name ); - case Node::OBJECT_NODE: return nodeGraph.CreateObjectNode( name ); - case Node::ALIAS_NODE: return nodeGraph.CreateAliasNode( name ); - case Node::EXE_NODE: return nodeGraph.CreateExeNode( name ); - case Node::CS_NODE: return nodeGraph.CreateCSNode( name ); - case Node::UNITY_NODE: return nodeGraph.CreateUnityNode( name ); - case Node::TEST_NODE: return nodeGraph.CreateTestNode( name ); - case Node::COMPILER_NODE: return nodeGraph.CreateCompilerNode( name ); - case Node::DLL_NODE: return nodeGraph.CreateDLLNode( name ); - case Node::VCXPROJECT_NODE: return nodeGraph.CreateVCXProjectNode( name ); - case Node::VSPROJEXTERNAL_NODE: return nodeGraph.CreateVSProjectExternalNode( name ); - case Node::OBJECT_LIST_NODE: return nodeGraph.CreateObjectListNode( name ); - case Node::COPY_DIR_NODE: return nodeGraph.CreateCopyDirNode( name ); - case Node::SLN_NODE: return nodeGraph.CreateSLNNode( name ); - case Node::REMOVE_DIR_NODE: return nodeGraph.CreateRemoveDirNode( name ); - case Node::XCODEPROJECT_NODE: return nodeGraph.CreateXCodeProjectNode( name ); - case Node::SETTINGS_NODE: return nodeGraph.CreateSettingsNode( name ); - case Node::TEXT_FILE_NODE: return nodeGraph.CreateTextFileNode( name ); - case Node::LIST_DEPENDENCIES_NODE: return nodeGraph.CreateListDependenciesNode( name ); - case Node::NUM_NODE_TYPES: ASSERT( false ); return nullptr; - } - - #if defined( __GNUC__ ) || defined( _MSC_VER ) - // GCC and incorrectly reports reaching end of non-void function (as of GCC 7.3.0) - // MSVC incorrectly reports reaching end of non-void function (as of VS 2017) - return nullptr; - #endif -} - // Load //------------------------------------------------------------------------------ /*static*/ Node * Node::Load( NodeGraph & nodeGraph, ConstMemoryStream & stream ) @@ -382,7 +342,7 @@ void Node::SetLastBuildTime( uint32_t ms ) VERIFY( stream.Read( name ) ); // Create node - Node * n = CreateNode( nodeGraph, (Type)nodeType, name ); + Node * n = nodeGraph.CreateNode( (Type)nodeType, name ); ASSERT( n ); // Early out for FileNode diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.h b/Code/Tools/FBuild/FBuildCore/Graph/Node.h index cb1deb0a3..25f80f878 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.h @@ -150,7 +150,6 @@ class Node : public Struct inline uint32_t GetProgressAccumulator() const { return m_ProgressAccumulator; } inline void SetProgressAccumulator( uint32_t p ) const { m_ProgressAccumulator = p; } - static Node * CreateNode( NodeGraph & nodeGraph, Node::Type nodeType, const AString & name ); static Node * Load( NodeGraph & nodeGraph, ConstMemoryStream & stream ); static void LoadDependencies( NodeGraph & nodeGraph, Node * node, ConstMemoryStream & stream ); static void Save( IOStream & stream, const Node * node ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index fe46dcf36..076c360af 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -213,7 +213,8 @@ bool NodeGraph::ParseFromRoot( const char * bffFile ) // default instance if needed. const AStackString<> settingsNodeName( "$$Settings$$" ); const Node * settingsNode = FindNode( settingsNodeName ); - m_Settings = settingsNode ? settingsNode->CastTo< SettingsNode >() : CreateSettingsNode( settingsNodeName ); // Create a default + m_Settings = settingsNode ? settingsNode->CastTo() + : CreateNode( settingsNodeName ); // Create a default // Parser will populate m_UsedFiles const Array & usedFiles = bffParser.GetUsedFiles(); @@ -794,302 +795,64 @@ void NodeGraph::RegisterNode( Node * node ) AddNode( node ); } -// CreateCopyFileNode +// CreateNode //------------------------------------------------------------------------------ -CopyFileNode * NodeGraph::CreateCopyFileNode( const AString & dstFileName ) +Node * NodeGraph::CreateNode( Node::Type type, const AString & name ) { ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( dstFileName ) ); - CopyFileNode * node = FNEW( CopyFileNode() ); - node->SetName( dstFileName ); - AddNode( node ); - return node; -} - -// CreateCopyDirNode -//------------------------------------------------------------------------------ -CopyDirNode * NodeGraph::CreateCopyDirNode( const AString & nodeName ) -{ - ASSERT( Thread::IsMainThread() ); - - CopyDirNode * node = FNEW( CopyDirNode() ); - node->SetName( nodeName ); - AddNode( node ); - return node; -} - -// CreateRemoveDirNode -//------------------------------------------------------------------------------ -RemoveDirNode * NodeGraph::CreateRemoveDirNode( const AString & nodeName ) -{ - ASSERT( Thread::IsMainThread() ); - - RemoveDirNode * node = FNEW( RemoveDirNode() ); - node->SetName( nodeName ); - AddNode( node ); - return node; -} - -// CreateExecNode -//------------------------------------------------------------------------------ -ExecNode * NodeGraph::CreateExecNode( const AString & nodeName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( nodeName ) ); - - ExecNode * node = FNEW( ExecNode() ); - node->SetName( nodeName ); - AddNode( node ); - return node; -} - -// CreateFileNode -//------------------------------------------------------------------------------ -FileNode * NodeGraph::CreateFileNode( const AString & fileName, bool cleanPath ) -{ - ASSERT( Thread::IsMainThread() ); - - FileNode * node; - - if ( cleanPath ) - { - AStackString< 512 > fullPath; - CleanPath( fileName, fullPath ); - node = FNEW( FileNode( fullPath, Node::FLAG_ALWAYS_BUILD ) ); - } - else - { - node = FNEW( FileNode( fileName, Node::FLAG_ALWAYS_BUILD ) ); - } - - AddNode( node ); - return node; -} - -// CreateDirectoryListNode -//------------------------------------------------------------------------------ -DirectoryListNode * NodeGraph::CreateDirectoryListNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - - DirectoryListNode * node = FNEW( DirectoryListNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateLibraryNode -//------------------------------------------------------------------------------ -LibraryNode * NodeGraph::CreateLibraryNode( const AString & libraryName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( libraryName ) ); - - LibraryNode * node = FNEW( LibraryNode() ); - node->SetName( libraryName ); - AddNode( node ); - return node; -} - -// CreateObjectNode -//------------------------------------------------------------------------------ -ObjectNode * NodeGraph::CreateObjectNode( const AString & objectName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( objectName ) ); - - ObjectNode * node = FNEW( ObjectNode() ); - node->SetName( objectName ); - AddNode( node ); - return node; -} - -// CreateAliasNode -//------------------------------------------------------------------------------ -AliasNode * NodeGraph::CreateAliasNode( const AString & aliasName ) -{ - ASSERT( Thread::IsMainThread() ); - - AliasNode * node = FNEW( AliasNode() ); - node->SetName( aliasName ); - AddNode( node ); - return node; -} - -// CreateDLLNode -//------------------------------------------------------------------------------ -DLLNode * NodeGraph::CreateDLLNode( const AString & dllName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( dllName ) ); - - DLLNode * node = FNEW( DLLNode() ); - node->SetName( dllName ); - AddNode( node ); - return node; -} - -// CreateExeNode -//------------------------------------------------------------------------------ -ExeNode * NodeGraph::CreateExeNode( const AString & exeName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( exeName ) ); - - ExeNode * node = FNEW( ExeNode() ); - node->SetName( exeName ); - AddNode( node ); - return node; -} - -// CreateUnityNode -//------------------------------------------------------------------------------ -UnityNode * NodeGraph::CreateUnityNode( const AString & unityName ) -{ - ASSERT( Thread::IsMainThread() ); - - UnityNode * node = FNEW( UnityNode() ); - node->SetName( unityName ); - AddNode( node ); - return node; -} - -// CreateCSNode -//------------------------------------------------------------------------------ -CSNode * NodeGraph::CreateCSNode( const AString & csAssemblyName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( csAssemblyName ) ); - - CSNode * node = FNEW( CSNode() ); - node->SetName( csAssemblyName ); - AddNode( node ); - return node; -} - -// CreateTestNode -//------------------------------------------------------------------------------ -TestNode * NodeGraph::CreateTestNode( const AString & testOutput ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( testOutput ) ); - - TestNode * node = FNEW( TestNode() ); - node->SetName( testOutput ); - AddNode( node ); - return node; -} - -// CreateCompilerNode -//------------------------------------------------------------------------------ -CompilerNode * NodeGraph::CreateCompilerNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - - CompilerNode * node = FNEW( CompilerNode() ); + Node * node = nullptr; + switch ( type ) + { + case Node::PROXY_NODE: ASSERT( false ); return nullptr; + case Node::COPY_FILE_NODE: node = FNEW( CopyFileNode() ); break; + case Node::DIRECTORY_LIST_NODE: node = FNEW( DirectoryListNode() ); break; + case Node::EXEC_NODE: node = FNEW( ExecNode() ); break; + case Node::FILE_NODE: return CreateFileNode( name ); // TODO: Eliminate special case + case Node::LIBRARY_NODE: node = FNEW( LibraryNode() ); break; + case Node::OBJECT_NODE: node = FNEW( ObjectNode() ); break; + case Node::ALIAS_NODE: node = FNEW( AliasNode() ); break; + case Node::EXE_NODE: node = FNEW( ExeNode() ); break; + case Node::CS_NODE: node = FNEW( CSNode() ); break; + case Node::UNITY_NODE: node = FNEW( UnityNode() ); break; + case Node::TEST_NODE: node = FNEW( TestNode() ); break; + case Node::COMPILER_NODE: node = FNEW( CompilerNode() ); break; + case Node::DLL_NODE: node = FNEW( DLLNode() ); break; + case Node::VCXPROJECT_NODE: node = FNEW( VCXProjectNode() ); break; + case Node::VSPROJEXTERNAL_NODE: node = FNEW( VSProjectExternalNode() ); break; + case Node::OBJECT_LIST_NODE: node = FNEW( ObjectListNode() ); break; + case Node::COPY_DIR_NODE: node = FNEW( CopyDirNode() ); break; + case Node::SLN_NODE: node = FNEW( SLNNode() ); break; + case Node::REMOVE_DIR_NODE: node = FNEW( RemoveDirNode() ); break; + case Node::XCODEPROJECT_NODE: node = FNEW( XCodeProjectNode() ); break; + case Node::SETTINGS_NODE: node = FNEW( SettingsNode() ); break; + case Node::TEXT_FILE_NODE: node = FNEW( TextFileNode() ); break; + case Node::LIST_DEPENDENCIES_NODE: node = FNEW( ListDependenciesNode() ); break; + case Node::NUM_NODE_TYPES: ASSERT( false ); return nullptr; + } + + ASSERT( node ); // All cases handled above means this is impossible + + // Names for files must be normalized by the time we get here + ASSERT( !node->IsAFile() || IsCleanPath( name ) ); + + // Store name and track new node node->SetName( name ); AddNode( node ); - return node; -} -// CreateVCXProjectNode -//------------------------------------------------------------------------------ -VSProjectBaseNode * NodeGraph::CreateVCXProjectNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( name ) ); - - VCXProjectNode * node = FNEW( VCXProjectNode() ); - node->SetName( name ); - AddNode( node ); return node; } -// CreateVSProjectExternalNode -//------------------------------------------------------------------------------ -VSProjectBaseNode * NodeGraph::CreateVSProjectExternalNode(const AString& name) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( name ) ); - - VSProjectExternalNode* node = FNEW( VSProjectExternalNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateSLNNode -//------------------------------------------------------------------------------ -SLNNode * NodeGraph::CreateSLNNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( name ) ); - - SLNNode * node = FNEW( SLNNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateObjectListNode -//------------------------------------------------------------------------------ -ObjectListNode * NodeGraph::CreateObjectListNode( const AString & listName ) -{ - ASSERT( Thread::IsMainThread() ); - - ObjectListNode * node = FNEW( ObjectListNode() ); - node->SetName( listName ); - AddNode( node ); - return node; -} - -// CreateXCodeProjectNode -//------------------------------------------------------------------------------ -XCodeProjectNode * NodeGraph::CreateXCodeProjectNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( name ) ); - - XCodeProjectNode * node = FNEW( XCodeProjectNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateSettingsNode -//------------------------------------------------------------------------------ -SettingsNode * NodeGraph::CreateSettingsNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - - SettingsNode * node = FNEW( SettingsNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateListDependenciesNode +// CreateFileNode //------------------------------------------------------------------------------ -ListDependenciesNode * NodeGraph::CreateListDependenciesNode( const AString & name ) +FileNode * NodeGraph::CreateFileNode( const AString & fileName ) { ASSERT( Thread::IsMainThread() ); - ListDependenciesNode * node = FNEW( ListDependenciesNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateTextFileNode -//------------------------------------------------------------------------------ -TextFileNode* NodeGraph::CreateTextFileNode( const AString& nodeName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( nodeName ) ); + AStackString< 512 > fullPath; + CleanPath( fileName, fullPath ); + FileNode * node = FNEW( FileNode( fullPath, Node::FLAG_ALWAYS_BUILD ) ); - TextFileNode* node = FNEW( TextFileNode() ); - node->SetName( nodeName ); AddNode( node ); return node; } @@ -2079,7 +1842,7 @@ void NodeGraph::MigrateNode( const NodeGraph & oldNodeGraph, Node & newNode, con else { // Create the dependency - newDepNode = Node::CreateNode( *this, oldDepNode->GetType(), oldDepNode->GetName() ); + newDepNode = CreateNode( oldDepNode->GetType(), oldDepNode->GetName() ); ASSERT( newDepNode ); newDeps.Add( newDepNode, oldDep.GetNodeStamp(), oldDep.IsWeak() ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index 59880707d..2db6d3a64 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -7,6 +7,7 @@ #include "Tools/FBuild/FBuildCore/BFF/BFFFileExists.h" #include "Tools/FBuild/FBuildCore/Helpers/SLNGenerator.h" #include "Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.h" +#include "Tools/FBuild/FBuildCore/Graph/Node.h" #include "Core/Containers/Array.h" #include "Core/Strings/AString.h" @@ -112,29 +113,13 @@ class NodeGraph void RegisterNode( Node * n ); // create new nodes - CopyFileNode * CreateCopyFileNode( const AString & dstFileName ); - CopyDirNode * CreateCopyDirNode( const AString & nodeName ); - RemoveDirNode * CreateRemoveDirNode( const AString & nodeName ); - ExecNode * CreateExecNode( const AString & dstFileName ); - FileNode * CreateFileNode( const AString & fileName, bool cleanPath = true ); - DirectoryListNode * CreateDirectoryListNode( const AString & name ); - LibraryNode * CreateLibraryNode( const AString & libraryName ); - ObjectNode * CreateObjectNode( const AString & objectName ); - AliasNode * CreateAliasNode( const AString & aliasName ); - DLLNode * CreateDLLNode( const AString & dllName ); - ExeNode * CreateExeNode( const AString & exeName ); - UnityNode * CreateUnityNode( const AString & unityName ); - CSNode * CreateCSNode( const AString & csAssemblyName ); - TestNode * CreateTestNode( const AString & testOutput ); - CompilerNode * CreateCompilerNode( const AString & name ); - VSProjectBaseNode * CreateVCXProjectNode( const AString & name ); - VSProjectBaseNode * CreateVSProjectExternalNode( const AString& name ); - SLNNode * CreateSLNNode( const AString & name ); - ObjectListNode * CreateObjectListNode( const AString & listName ); - XCodeProjectNode * CreateXCodeProjectNode( const AString & name ); - SettingsNode * CreateSettingsNode( const AString & name ); - ListDependenciesNode* CreateListDependenciesNode( const AString& name ); - TextFileNode * CreateTextFileNode( const AString & name ); + Node * CreateNode( Node::Type type, const AString & name ); + template + T * CreateNode( const AString & name ) + { + return CreateNode( T::GetTypeS(), name )->template CastTo(); + } + FileNode * CreateFileNode( const AString & fileName ); void DoBuildPass( Node * nodeToBuild ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index e5649bcf5..75570e49a 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -777,7 +777,7 @@ ObjectNode * ObjectListNode::CreateObjectNode( NodeGraph & nodeGraph, const AString & objectInput, const AString & pchObjectName ) { - ObjectNode * node= nodeGraph.CreateObjectNode( objectName ); + ObjectNode * node= nodeGraph.CreateNode( objectName ); node->m_Compiler = m_Compiler; node->m_CompilerOptions = compilerOptions; node->m_CompilerOptionsDeoptimized = compilerOptionsDeoptimized; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp index 3357a908c..de6136319 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp @@ -46,7 +46,7 @@ void TestExe::CreateNode() const #else AStackString<> exeName( "/tmp/exe.exe" ); #endif - const ExeNode * exeNode = ng.CreateExeNode( exeName ); + const ExeNode * exeNode = ng.CreateNode( exeName ); TEST_ASSERT( exeNode->GetType() == Node::EXE_NODE ); TEST_ASSERT( ExeNode::GetTypeS() == Node::EXE_NODE ); TEST_ASSERT( AStackString<>( "Exe" ) == exeNode->GetTypeName() ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp index 28cadff92..9d1a0275c 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp @@ -19,7 +19,9 @@ #include "Tools/FBuild/FBuildCore/Graph/LibraryNode.h" #include "Tools/FBuild/FBuildCore/Graph/NodeGraph.h" #include "Tools/FBuild/FBuildCore/Graph/ObjectNode.h" +#include "Tools/FBuild/FBuildCore/Graph/RemoveDirNode.h" #include "Tools/FBuild/FBuildCore/Graph/SettingsNode.h" +#include "Tools/FBuild/FBuildCore/Graph/TestNode.h" #include "Tools/FBuild/FBuildCore/Graph/UnityNode.h" // Core @@ -118,114 +120,41 @@ void TestGraph::TestNodeTypes() const FBuild fb; NodeGraph ng; - const FileNode * fn = ng.CreateFileNode( AStackString<>( "file" ) ); - TEST_ASSERT( fn->GetType() == Node::FILE_NODE ); - TEST_ASSERT( FileNode::GetTypeS() == Node::FILE_NODE ); - - { - #if defined( __WINDOWS__ ) - const CompilerNode * cn = ng.CreateCompilerNode( AStackString<>( "c:\\cl.exe" ) ); - #else - const CompilerNode * cn = ng.CreateCompilerNode( AStackString<>( "/usr/bin/gcc" ) ); - #endif - TEST_ASSERT( cn->GetType() == Node::COMPILER_NODE ); - TEST_ASSERT( AStackString<>( "Compiler" ) == cn->GetTypeName() ); - } - - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateCopyFileNode( AStackString<>( "c:\\dummy" ) ); - #else - const Node * n = ng.CreateCopyFileNode( AStackString<>( "/dummy/dummy" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::COPY_FILE_NODE ); - TEST_ASSERT( CopyFileNode::GetTypeS() == Node::COPY_FILE_NODE ); - TEST_ASSERT( AStackString<>( "CopyFile" ) == n->GetTypeName() ); - } - + // Node names differ on Window vs other platforms due to paths etc #if defined( __WINDOWS__ ) - const DirectoryListNode * dn = ng.CreateDirectoryListNode( AStackString<>( "path\\|*.cpp|false|" ) ); + #define CHOOSE_NAME( WINDOWS_PATH, OTHER_PATH ) AStackString<> name( WINDOWS_PATH ) #else - const DirectoryListNode * dn = ng.CreateDirectoryListNode( AStackString<>( "path/|*.cpp|false|" ) ); + #define CHOOSE_NAME( WINDOWS_PATH, OTHER_PATH ) AStackString<> name( OTHER_PATH ) #endif - TEST_ASSERT( dn->GetType() == Node::DIRECTORY_LIST_NODE ); - TEST_ASSERT( DirectoryListNode::GetTypeS() == Node::DIRECTORY_LIST_NODE ); - TEST_ASSERT( AStackString<>( "Directory" ) == dn->GetTypeName() ); - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateExecNode( AStackString<>( "c:\\execdummy" ) ); - #else - const Node * n = ng.CreateExecNode( AStackString<>( "/execdummy/execdummy" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::EXEC_NODE ); - TEST_ASSERT( ExecNode::GetTypeS() == Node::EXEC_NODE ); - TEST_ASSERT( AStackString<>( "Exec" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateLibraryNode( AStackString<>( "c:\\library.lib" ) ); - #else - const Node * n = ng.CreateLibraryNode( AStackString<>( "/library/library.a" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::LIBRARY_NODE ); - TEST_ASSERT( LibraryNode::GetTypeS() == Node::LIBRARY_NODE ); - TEST_ASSERT( AStackString<>( "Library" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateObjectNode( AStackString<>( "c:\\object.lib" ) ); - #else - const Node * n = ng.CreateObjectNode( AStackString<>( "/library/object.o" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::OBJECT_NODE ); - TEST_ASSERT( ObjectNode::GetTypeS() == Node::OBJECT_NODE ); - TEST_ASSERT( AStackString<>( "Object" ) == n->GetTypeName() ); - } - { - const Node * n = ng.CreateAliasNode( AStackString<>( "alias" ) ); - TEST_ASSERT( n->GetType() == Node::ALIAS_NODE ); - TEST_ASSERT( AliasNode::GetTypeS() == Node::ALIAS_NODE ); - TEST_ASSERT( AStackString<>( "Alias" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - AStackString<> dllName( "c:\\lib.dll" ); - #else - AStackString<> dllName( "/tmp/lib.so" ); - #endif - const Node * n = ng.CreateDLLNode( dllName ); - TEST_ASSERT( n->GetType() == Node::DLL_NODE ); - TEST_ASSERT( DLLNode::GetTypeS() == Node::DLL_NODE ); - TEST_ASSERT( AStackString<>( "DLL" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - AStackString<> exeName( "c:\\exe.exe" ); - #else - AStackString<> exeName( "/tmp/exe.exe" ); - #endif - const Node * n = ng.CreateExeNode( exeName ); - TEST_ASSERT( n->GetType() == Node::EXE_NODE ); - TEST_ASSERT( ExeNode::GetTypeS() == Node::EXE_NODE ); - TEST_ASSERT( AStackString<>( "Exe" ) == n->GetTypeName() ); - } - { - const Node * n = ng.CreateUnityNode( AStackString<>( "Unity" ) ); - TEST_ASSERT( n->GetType() == Node::UNITY_NODE ); - TEST_ASSERT( UnityNode::GetTypeS() == Node::UNITY_NODE ); - TEST_ASSERT( AStackString<>( "Unity" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateCSNode( AStackString<>( "c:\\csharp.dll" ) ); - #else - const Node * n = ng.CreateCSNode( AStackString<>( "/dummy/csharp.dll" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::CS_NODE); - TEST_ASSERT( CSNode::GetTypeS() == Node::CS_NODE ); - TEST_ASSERT( AStackString<>( "C#" ) == n->GetTypeName() ); - } + // Test each node can be created and type mappings are consistent + #define TEST_NODE( TYPE, TYPE_ENUM, FRIENDLY_TYPE, WINDOWS_PATH, OTHER_PATH ) \ + { \ + CHOOSE_NAME( WINDOWS_PATH, OTHER_PATH ); \ + const TYPE * node = ng.CreateNode( name ); \ + TEST_ASSERT( node->GetType() == Node::TYPE_ENUM ); \ + TEST_ASSERT( TYPE::GetTypeS() == Node::TYPE_ENUM ); \ + TEST_ASSERT( AStackString<>( FRIENDLY_TYPE ) == node->GetTypeName() ); \ + } while( false ) + + // TODO:C - It would be nice to restructure this so that new nodes are automatically tested + TEST_NODE( FileNode, FILE_NODE, "File", "file", "file" ); + TEST_NODE( CompilerNode, COMPILER_NODE, "Compiler", "c:\\cl.exe", "/usr/bin/gcc" ); + TEST_NODE( CopyFileNode, COPY_FILE_NODE, "CopyFile", "c:\\file", "/path/file" ); + TEST_NODE( DirectoryListNode, DIRECTORY_LIST_NODE, "Directory", "path\\|*.cpp|false|", "path/|*.cpp|false|" ); + TEST_NODE( ExecNode, EXEC_NODE, "Exec", "c:\\exec", "/path/exec" ); + TEST_NODE( LibraryNode, LIBRARY_NODE, "Library", "c:\\library.lib", "/library/library.a" ); + TEST_NODE( ObjectNode, OBJECT_NODE, "Object", "c:\\object.obj", "/path/object.a" ); + TEST_NODE( AliasNode, ALIAS_NODE, "Alias", "alias", "alias" ); + TEST_NODE( DLLNode, DLL_NODE, "DLL", "c:\\lib.dll", "/tmp/lib.so" ); + TEST_NODE( ExeNode, EXE_NODE, "Exe", "c:\\exe.exe", "/tmp/exe.exe" ); + TEST_NODE( UnityNode, UNITY_NODE, "Unity", "unity", "unity" ); + TEST_NODE( CSNode, CS_NODE, "C#", "c:\\csharp.dll", "/path/csharp.dll" ); + TEST_NODE( TestNode, TEST_NODE, "Test", "c:\\output.txt", "/path/output.txt" ); + TEST_NODE( RemoveDirNode, REMOVE_DIR_NODE, "RemoveDir", "remove", "remove" ); + + #undef TEST_NODE + #undef CHOOSE_NAME } // FileNode @@ -293,7 +222,7 @@ void TestGraph::TestDirectoryListNode() const name ); // create the node, and make sure we can access it by name - DirectoryListNode * node = ng.CreateDirectoryListNode( name ); + DirectoryListNode * node = ng.CreateNode( name ); node->m_Path = testFolder; node->m_Patterns = patterns; const BFFToken * token = nullptr; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestRemoveDir.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestRemoveDir.cpp index 434bea54d..a78264fb3 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestRemoveDir.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestRemoveDir.cpp @@ -19,31 +19,15 @@ class TestRemoveDir : public FBuildTest private: DECLARE_TESTS - void CreateNode() const; void RemoveDir() const; }; // Register Tests //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestRemoveDir ) - REGISTER_TEST( CreateNode ) REGISTER_TEST( RemoveDir ) REGISTER_TESTS_END -// CreateNode -//------------------------------------------------------------------------------ -void TestRemoveDir::CreateNode() const -{ - FBuild fb; - NodeGraph ng; - - const RemoveDirNode * removeDirNode = ng.CreateRemoveDirNode( AStackString<>( "name" ) ); - - TEST_ASSERT( removeDirNode->GetType() == Node::REMOVE_DIR_NODE ); - TEST_ASSERT( RemoveDirNode::GetTypeS() == Node::REMOVE_DIR_NODE ); - TEST_ASSERT( AStackString<>( "RemoveDir" ) == removeDirNode->GetTypeName() ); -} - // RemoveDir //------------------------------------------------------------------------------ void TestRemoveDir::RemoveDir() const diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestTest.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestTest.cpp index 3d219c1c4..352e8cbf9 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestTest.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestTest.cpp @@ -20,7 +20,6 @@ class TestTest : public FBuildTest private: DECLARE_TESTS - void CreateNode() const; void Build() const; void Build_NoRebuild() const; void Fail_ReturnCode() const; @@ -32,7 +31,6 @@ class TestTest : public FBuildTest // Register Tests //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestTest ) - REGISTER_TEST( CreateNode ) REGISTER_TEST( Build ) REGISTER_TEST( Build_NoRebuild ) REGISTER_TEST( Fail_ReturnCode ) @@ -41,22 +39,6 @@ REGISTER_TESTS_BEGIN( TestTest ) REGISTER_TEST( Exclusions ) REGISTER_TESTS_END -// CreateNode -//------------------------------------------------------------------------------ -void TestTest::CreateNode() const -{ - FBuild fb; - NodeGraph ng; - - AStackString<> outputPath; - NodeGraph::CleanPath( AStackString<>( "output.txt" ), outputPath ); - const TestNode * testNode = ng.CreateTestNode( outputPath ); - - TEST_ASSERT( testNode->GetType() == Node::TEST_NODE ); - TEST_ASSERT( TestNode::GetTypeS() == Node::TEST_NODE ); - TEST_ASSERT( AStackString<>( "Test" ) == testNode->GetTypeName() ); -} - // Build //------------------------------------------------------------------------------ void TestTest::Build() const From 16a9e1629ec4c6e750ac6f5b532bcb17035959b4 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Mon, 3 Jul 2023 12:55:57 +0930 Subject: [PATCH 043/129] [Improvement] Improve database load time slightly - Avoid double handling of node names - Avoid unnecessary path normalization during loading (paths saved are already normalized) - Simplify node creation by eliminating unnecessary args passing - Remove CreateFileNode (Intergrate special case into regular CreateNode() function. Wliminating this special case entirely is still desirable and abstracting the special case away from callers is a step towards that. --- .../FBuildCore/BFF/Functions/Function.cpp | 10 ++--- .../FBuildCore/BFF/Functions/FunctionCopy.cpp | 2 +- .../FBuild/FBuildCore/Graph/AliasNode.cpp | 3 +- Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp | 4 +- .../FBuild/FBuildCore/Graph/CompilerNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/CopyDirNode.cpp | 4 +- .../FBuild/FBuildCore/Graph/CopyFileNode.cpp | 2 +- .../FBuildCore/Graph/DirectoryListNode.cpp | 3 +- .../FBuild/FBuildCore/Graph/ExecNode.cpp | 4 +- .../FBuild/FBuildCore/Graph/FileNode.cpp | 16 ++++---- Code/Tools/FBuild/FBuildCore/Graph/FileNode.h | 2 +- .../FBuild/FBuildCore/Graph/LinkerNode.cpp | 6 +-- .../FBuildCore/Graph/ListDependenciesNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/Node.cpp | 13 +++--- Code/Tools/FBuild/FBuildCore/Graph/Node.h | 6 +-- .../FBuild/FBuildCore/Graph/NodeGraph.cpp | 41 ++++++++++++++----- .../Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 2 +- .../FBuild/FBuildCore/Graph/NodeProxy.cpp | 5 ++- .../Tools/FBuild/FBuildCore/Graph/NodeProxy.h | 2 +- .../FBuildCore/Graph/ObjectListNode.cpp | 8 ++-- .../FBuild/FBuildCore/Graph/ObjectNode.cpp | 21 +++++----- .../FBuild/FBuildCore/Graph/ObjectNode.h | 2 +- .../FBuild/FBuildCore/Graph/RemoveDirNode.cpp | 3 +- .../Tools/FBuild/FBuildCore/Graph/SLNNode.cpp | 3 +- .../FBuild/FBuildCore/Graph/SettingsNode.cpp | 6 +-- .../FBuild/FBuildCore/Graph/TestNode.cpp | 4 +- .../FBuild/FBuildCore/Graph/TextFileNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/UnityNode.cpp | 2 +- .../FBuildCore/Graph/VSProjectBaseNode.cpp | 3 +- .../FBuildCore/Graph/XCodeProjectNode.cpp | 3 +- .../FBuild/FBuildTest/Tests/TestGraph.cpp | 10 +++-- .../FBuildTest/Tests/TestNodeReflection.cpp | 6 ++- 32 files changed, 115 insertions(+), 87 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp index e4c0c3a3e..ebeca26e9 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp @@ -315,14 +315,14 @@ Function::~Function() = default; // - For nodes that specify a name as a property (usually the output of the node) // use that as the name (i.e. the filename is the name) // - Otherwise, what would normally be the alias - AStackString<> nameFromMetaData; + AString nameFromMetaData; if ( GetNameForNode( nodeGraph, funcStartIter, node->GetReflectionInfoV(), nameFromMetaData ) == false ) { FDELETE node; return false; // GetNameForNode will have emitted an error } const bool aliasUsedForName = nameFromMetaData.IsEmpty(); - const AString & name = ( aliasUsedForName ) ? m_AliasForFunction : nameFromMetaData; + AString & name = ( aliasUsedForName ) ? m_AliasForFunction : nameFromMetaData; ASSERT( name.IsEmpty() == false ); // Check name isn't already used @@ -334,7 +334,7 @@ Function::~Function() = default; } // Set Name - node->SetName( name ); + node->SetName( Move( name ) ); // Register with NodeGraph nodeGraph.RegisterNode( node ); @@ -676,7 +676,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, Node * node = nodeGraph.FindNode( file ); if ( node == nullptr ) { - node = nodeGraph.CreateFileNode( file ); + node = nodeGraph.CreateNode( file ); } else if ( node->IsAFile() == false ) { @@ -786,7 +786,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, if ( n == nullptr ) { // not found - create a new file node - n = nodeGraph.CreateFileNode( nodeName ); + n = nodeGraph.CreateNode( nodeName ); nodes.Add( n ); return true; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp index 102f8aca1..7feebc2f7 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp @@ -104,7 +104,7 @@ FunctionCopy::FunctionCopy() else { // source file not defined by use - assume an external file - srcNodes.Append( nodeGraph.CreateFileNode( *it ) ); + srcNodes.Append( nodeGraph.CreateNode( *it ) ); } } } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/AliasNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/AliasNode.cpp index f53b2a483..baa3d4477 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/AliasNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/AliasNode.cpp @@ -20,8 +20,9 @@ REFLECT_END( AliasNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ AliasNode::AliasNode() - : Node( AString::GetEmpty(), Node::ALIAS_NODE, Node::FLAG_ALWAYS_BUILD ) + : Node( Node::ALIAS_NODE ) { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 1; // almost no work is done for this node } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp index d98ef4bd1..12a220a3d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp @@ -42,7 +42,7 @@ REFLECT_END( CSNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ CSNode::CSNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_CompilerInputPathRecurse( true ) , m_NumCompilerInputFiles( 0 ) , m_NumCompilerReferences( 0 ) @@ -151,7 +151,7 @@ CSNode::~CSNode() = default; Node * sn = nodeGraph.FindNode( file.m_Name ); if ( sn == nullptr ) { - sn = nodeGraph.CreateFileNode( file.m_Name ); + sn = nodeGraph.CreateNode( file.m_Name ); } else if ( sn->IsAFile() == false ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp index d552e1e85..9d8e41847 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp @@ -43,7 +43,7 @@ REFLECT_END( CompilerNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ CompilerNode::CompilerNode() - : Node( AString::GetEmpty(), Node::COMPILER_NODE, Node::FLAG_NONE ) + : Node( Node::COMPILER_NODE ) , m_AllowDistribution( true ) , m_AllowResponseFile( false ) , m_ForceResponseFile( false ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp index 3541c1fd5..162af65ec 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp @@ -29,7 +29,7 @@ REFLECT_END( CopyDirNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ CopyDirNode::CopyDirNode() -: Node( AString::GetEmpty(), Node::COPY_DIR_NODE, Node::FLAG_NONE ) + : Node( Node::COPY_DIR_NODE ) { } @@ -119,7 +119,7 @@ CopyDirNode::~CopyDirNode() = default; Node * srcFileNode = nodeGraph.FindNode( srcFile ); if ( srcFileNode == nullptr ) { - srcFileNode = nodeGraph.CreateFileNode( srcFile ); + srcFileNode = nodeGraph.CreateNode( srcFile ); } else if ( srcFileNode->IsAFile() == false ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyFileNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyFileNode.cpp index 9abddd8d4..cc9c65ef6 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyFileNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyFileNode.cpp @@ -25,7 +25,7 @@ REFLECT_END( CopyFileNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ CopyFileNode::CopyFileNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() { m_Type = Node::COPY_FILE_NODE; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp index 783ac27b8..e97976a27 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp @@ -31,10 +31,11 @@ REFLECT_END( DirectoryListNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ DirectoryListNode::DirectoryListNode() - : Node( AString::GetEmpty(), Node::DIRECTORY_LIST_NODE, Node::FLAG_ALWAYS_BUILD ) + : Node( Node::DIRECTORY_LIST_NODE ) , m_Recursive( true ) , m_IncludeReadOnlyStatusInHash( false ) { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 100; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp index ad0cef1ce..8b9b28bd3 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp @@ -45,7 +45,7 @@ REFLECT_END( ExecNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ ExecNode::ExecNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_ExecReturnCode( 0 ) , m_ExecAlwaysShowOutput( false ) , m_ExecUseStdOutAsOutput( false ) @@ -145,7 +145,7 @@ ExecNode::~ExecNode() Node * sn = nodeGraph.FindNode( file.m_Name ); if ( sn == nullptr ) { - sn = nodeGraph.CreateFileNode( file.m_Name ); + sn = nodeGraph.CreateNode( file.m_Name ); } else if ( sn->IsAFile() == false ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/FileNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/FileNode.cpp index f03677017..7c780dd57 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/FileNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/FileNode.cpp @@ -16,15 +16,9 @@ // CONSTRUCTOR //------------------------------------------------------------------------------ -FileNode::FileNode( const AString & fileName, uint8_t controlFlags ) - : Node( fileName, Node::FILE_NODE, controlFlags ) +FileNode::FileNode() + : Node( Node::FILE_NODE ) { - ASSERT( fileName.EndsWith( "\\" ) == false ); - #if defined( __WINDOWS__ ) - ASSERT( ( fileName.FindLast( ':' ) == nullptr ) || - ( fileName.FindLast( ':' ) == ( fileName.Get() + 1 ) ) ); - #endif - m_LastBuildTimeMs = 1; // very little work required } @@ -44,6 +38,12 @@ FileNode::~FileNode() = default; //------------------------------------------------------------------------------ /*virtual*/ Node::BuildResult FileNode::DoBuild( Job * /*job*/ ) { + ASSERT( m_Name.EndsWith( "\\" ) == false ); + #if defined( __WINDOWS__ ) + ASSERT( ( m_Name.FindLast( ':' ) == nullptr ) || + ( m_Name.FindLast( ':' ) == ( m_Name.Get() + 1 ) ) ); + #endif + // NOTE: Not calling RecordStampFromBuiltFile as this is not a built file m_Stamp = FileIO::GetFileLastWriteTime( m_Name ); // Don't assert m_Stamp != 0 as input file might not exist diff --git a/Code/Tools/FBuild/FBuildCore/Graph/FileNode.h b/Code/Tools/FBuild/FBuildCore/Graph/FileNode.h index 2f7cfe3f4..6312ccaff 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/FileNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/FileNode.h @@ -11,7 +11,7 @@ class FileNode : public Node { public: - explicit FileNode( const AString & fileName, uint8_t controlFlags ); + FileNode(); virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * funcStartIter, const Function * function ) override; virtual ~FileNode() override; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp index 28bd76f8b..953e7d357 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp @@ -56,7 +56,7 @@ REFLECT_END( LinkerNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ LinkerNode::LinkerNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_LinkerType( "auto" ) , m_LinkerAllowResponseFile( false ) , m_LinkerForceResponseFile( false ) @@ -1378,7 +1378,7 @@ void LinkerNode::GetImportLibName( const AString & args, AString & importLibName // see if the file exists on disk at this location if ( LinkerNodeFileExistsCache::Get().FileExists( potentialNodeNameClean ) ) { - node = nodeGraph.CreateFileNode( potentialNodeNameClean ); + node = nodeGraph.CreateNode( potentialNodeNameClean ); libs.Add( node ); found = true; FLOG_VERBOSE( "Additional library '%s' assumed to be '%s'\n", lib.Get(), potentialNodeNameClean.Get() ); @@ -1519,7 +1519,7 @@ void LinkerNode::GetImportLibName( const AString & args, AString & importLibName // node not found - create a new FileNode, assuming we are // linking against an externally built library - node = nodeGraph.CreateFileNode( nodeName ); + node = nodeGraph.CreateNode( nodeName ); nodes.Add( node ); return true; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ListDependenciesNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ListDependenciesNode.cpp index 09892edc1..f21b9df48 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ListDependenciesNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ListDependenciesNode.cpp @@ -87,7 +87,7 @@ class DependencyAscendingCompareIDeref // CONSTRUCTOR //------------------------------------------------------------------------------ ListDependenciesNode::ListDependenciesNode() -: FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() { m_Type = Node::LIST_DEPENDENCIES_NODE; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp index 92fe09d31..1b7ead001 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp @@ -122,12 +122,9 @@ REFLECT_END( Node ) // CONSTRUCTOR //------------------------------------------------------------------------------ -Node::Node( const AString & name, Type type, uint8_t controlFlags ) +Node::Node( Type type ) { m_Type = type; - m_ControlFlags = controlFlags; - - SetName( name ); // Compile time check to ensure name vector is in sync static_assert( sizeof( s_NodeTypeNames ) / sizeof(const char *) == NUM_NODE_TYPES, "s_NodeTypeNames item count doesn't match NUM_NODE_TYPES" ); @@ -338,11 +335,11 @@ void Node::SetLastBuildTime( uint32_t ms ) PROFILE_SECTION( Node::GetTypeName( (Type)nodeType ) ); // Name of node - AStackString<> name; + AString name; VERIFY( stream.Read( name ) ); // Create node - Node * n = nodeGraph.CreateNode( (Type)nodeType, name ); + Node * n = nodeGraph.CreateNode( (Type)nodeType, Move( name ) ); ASSERT( n ); // Early out for FileNode @@ -716,10 +713,10 @@ void Node::SetLastBuildTime( uint32_t ms ) // SetName //------------------------------------------------------------------------------ -void Node::SetName( const AString & name ) +void Node::SetName( AString && name ) { - m_Name = name; m_NameCRC = CRC32::CalcLower( name ); + m_Name = Move( name ); } // ReplaceDummyName diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.h b/Code/Tools/FBuild/FBuildCore/Graph/Node.h index 25f80f878..be53125f8 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.h @@ -123,7 +123,7 @@ class Node : public Struct UP_TO_DATE, // built, or confirmed as not needing building }; - explicit Node( const AString & name, Type type, uint8_t controlFlags ); + explicit Node( Type type ); virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * funcStartIter, const Function * function ) = 0; virtual ~Node(); @@ -196,7 +196,7 @@ class Node : public Struct friend class WorkerThread; friend class CompilationDatabase; - void SetName( const AString & name ); + void SetName( AString && name ); void ReplaceDummyName( const AString & newName ); @@ -251,7 +251,7 @@ class Node : public Struct mutable uint16_t m_StatsFlags = 0; // Stats recorded in the current build mutable uint32_t m_BuildPassTag = 0; // Prevent multiple recursions into the same node during a single sweep uint64_t m_Stamp = 0; // "Stamp" representing this node for dependency comparissons - uint8_t m_ControlFlags; // Control build behavior special cases - Set by constructor + uint8_t m_ControlFlags = FLAG_NONE; // Control build behavior special cases - Set by constructor bool m_Hidden = false; // Hidden from -showtargets? // Note: Unused 2 bytes here uint32_t m_RecursiveCost = 0; // Recursive cost used during task ordering diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index 076c360af..48d04c2f7 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -797,7 +797,7 @@ void NodeGraph::RegisterNode( Node * node ) // CreateNode //------------------------------------------------------------------------------ -Node * NodeGraph::CreateNode( Node::Type type, const AString & name ) +Node * NodeGraph::CreateNode( Node::Type type, AString && name ) { ASSERT( Thread::IsMainThread() ); @@ -808,7 +808,12 @@ Node * NodeGraph::CreateNode( Node::Type type, const AString & name ) case Node::COPY_FILE_NODE: node = FNEW( CopyFileNode() ); break; case Node::DIRECTORY_LIST_NODE: node = FNEW( DirectoryListNode() ); break; case Node::EXEC_NODE: node = FNEW( ExecNode() ); break; - case Node::FILE_NODE: return CreateFileNode( name ); // TODO: Eliminate special case + case Node::FILE_NODE: + { + node = FNEW( FileNode() ); + node->m_ControlFlags = Node::FLAG_ALWAYS_BUILD; // TODO:C Eliminate special case + break; + } case Node::LIBRARY_NODE: node = FNEW( LibraryNode() ); break; case Node::OBJECT_NODE: node = FNEW( ObjectNode() ); break; case Node::ALIAS_NODE: node = FNEW( AliasNode() ); break; @@ -837,24 +842,38 @@ Node * NodeGraph::CreateNode( Node::Type type, const AString & name ) ASSERT( !node->IsAFile() || IsCleanPath( name ) ); // Store name and track new node - node->SetName( name ); + node->SetName( Move( name ) ); AddNode( node ); return node; } -// CreateFileNode + +// CreateNode //------------------------------------------------------------------------------ -FileNode * NodeGraph::CreateFileNode( const AString & fileName ) +Node * NodeGraph::CreateNode( Node::Type type, const AString & name ) { - ASSERT( Thread::IsMainThread() ); + // Where possible callers should call the move version to transfer ownership + // of strings, but calers don't always have a string to transfer so this + // helper can be called in those situations + AString nameCopy; - AStackString< 512 > fullPath; - CleanPath( fileName, fullPath ); - FileNode * node = FNEW( FileNode( fullPath, Node::FLAG_ALWAYS_BUILD ) ); + // TODO:C Eliminate special case handling of FileNode + // For historical reasons users of FileNodes don't clean paths so we have to + // do it here. Ideally this special case would be eliminated in the future. + if ( type == Node::FILE_NODE ) + { + // Clean path + AStackString< 512 > cleanPath; + CleanPath( name, cleanPath ); + nameCopy = cleanPath; + } + else + { + nameCopy = name; + } - AddNode( node ); - return node; + return CreateNode( type, Move( nameCopy ) ); } // AddNode diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index 2db6d3a64..36ed1ae24 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -113,13 +113,13 @@ class NodeGraph void RegisterNode( Node * n ); // create new nodes + Node * CreateNode( Node::Type type, AString && name ); Node * CreateNode( Node::Type type, const AString & name ); template T * CreateNode( const AString & name ) { return CreateNode( T::GetTypeS(), name )->template CastTo(); } - FileNode * CreateFileNode( const AString & fileName ); void DoBuildPass( Node * nodeToBuild ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.cpp index ab6f13dda..b12900b55 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.cpp @@ -7,9 +7,10 @@ // CONSTRUCTOR //------------------------------------------------------------------------------ -NodeProxy::NodeProxy( const AString & name ) - : Node( name, Node::PROXY_NODE, 0 ) +NodeProxy::NodeProxy( AString && name ) + : Node( Node::PROXY_NODE ) { + SetName( Move( name ) ); } // DESTRUCTOR diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.h index 3c05f96c2..c33c0197a 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.h @@ -11,7 +11,7 @@ class NodeProxy : public Node { public: - explicit NodeProxy( const AString & name ); + explicit NodeProxy( AString && name ); virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * funcStartIter, const Function * function ) override; virtual ~NodeProxy() override; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index 75570e49a..25565a979 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -77,7 +77,7 @@ REFLECT_END( ObjectListNode ) // ObjectListNode //------------------------------------------------------------------------------ ObjectListNode::ObjectListNode() -: Node( AString::GetEmpty(), Node::OBJECT_LIST_NODE, Node::FLAG_NONE ) + : Node( Node::OBJECT_LIST_NODE ) { m_LastBuildTimeMs = 10000; @@ -371,7 +371,7 @@ ObjectListNode::~ObjectListNode() = default; Node * n = nodeGraph.FindNode( fIt->m_Name ); if ( n == nullptr ) { - n = nodeGraph.CreateFileNode( fIt->m_Name ); + n = nodeGraph.CreateNode( fIt->m_Name ); } else if ( n->IsAFile() == false ) { @@ -409,7 +409,7 @@ ObjectListNode::~ObjectListNode() = default; Node * n = nodeGraph.FindNode( *it ); if ( n == nullptr ) { - n = nodeGraph.CreateFileNode( *it ); + n = nodeGraph.CreateNode( *it ); } else if ( n->IsAFile() == false ) { @@ -430,7 +430,7 @@ ObjectListNode::~ObjectListNode() = default; Node * n = nodeGraph.FindNode( isolatedFile.GetFileName() ); if ( n == nullptr ) { - n = nodeGraph.CreateFileNode( isolatedFile.GetFileName() ); + n = nodeGraph.CreateNode( isolatedFile.GetFileName() ); } else if ( n->IsAFile() == false ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp index 0b139f1df..8390e6cfe 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp @@ -95,7 +95,7 @@ REFLECT_END( ObjectNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ ObjectNode::ObjectNode() -: FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() { m_Type = OBJECT_NODE; m_LastBuildTimeMs = 5000; // higher default than a file node @@ -170,14 +170,15 @@ ObjectNode::ObjectNode() // CONSTRUCTOR (Remote) //------------------------------------------------------------------------------ -ObjectNode::ObjectNode( const AString & objectName, +ObjectNode::ObjectNode( AString && objectName, NodeProxy * srcFile, const AString & compilerOptions, uint32_t flags ) -: FileNode( objectName, Node::FLAG_NONE ) -, m_CompilerOptions( compilerOptions ) -, m_Remote( true ) + : FileNode() + , m_CompilerOptions( compilerOptions ) + , m_Remote( true ) { + SetName( Move( objectName ) ); m_Type = OBJECT_NODE; m_LastBuildTimeMs = 5000; // higher default than a file node m_CompilerFlags.m_Flags = flags; @@ -294,7 +295,7 @@ ObjectNode::~ObjectNode() Node * fn = nodeGraph.FindNode( *it ); if ( fn == nullptr ) { - fn = nodeGraph.CreateFileNode( *it ); + fn = nodeGraph.CreateNode( *it ); } else if ( fn->IsAFile() == false ) { @@ -893,8 +894,8 @@ bool ObjectNode::ProcessIncludesWithPreProcessor( Job * job ) //------------------------------------------------------------------------------ /*static*/ Node * ObjectNode::LoadRemote( IOStream & stream ) { - AStackString<> name; - AStackString<> sourceFile; + AString name; + AString sourceFile; uint32_t flags; AStackString<> compilerArgs; if ( ( stream.Read( name ) == false ) || @@ -905,9 +906,9 @@ bool ObjectNode::ProcessIncludesWithPreProcessor( Job * job ) return nullptr; } - NodeProxy * srcFile = FNEW( NodeProxy( sourceFile ) ); + NodeProxy * srcFile = FNEW( NodeProxy( Move( sourceFile ) ) ); - return FNEW( ObjectNode( name, srcFile, compilerArgs, flags ) ); + return FNEW( ObjectNode( Move( name ), srcFile, compilerArgs, flags ) ); } // DetermineFlags diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.h b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.h index f8ce67c64..ce2784c42 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.h @@ -41,7 +41,7 @@ class ObjectNode : public FileNode ObjectNode(); virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * iter, const Function * function ) override; // simplified remote constructor - explicit ObjectNode( const AString & objectName, + explicit ObjectNode( AString && objectName, NodeProxy * srcFile, const AString & compilerOptions, uint32_t flags ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp index a2840b840..56905ab8d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp @@ -28,9 +28,10 @@ REFLECT_END( RemoveDirNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ RemoveDirNode::RemoveDirNode() - : Node( AString::GetEmpty(), Node::REMOVE_DIR_NODE, Node::FLAG_ALWAYS_BUILD ) + : Node( Node::REMOVE_DIR_NODE ) , m_RemovePathsRecurse( true ) { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_RemovePatterns.EmplaceBack( "*" ); } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp index b6a85ba5d..1e57dbae0 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp @@ -76,8 +76,9 @@ struct VCXProjectNodeComp // CONSTRUCTOR //------------------------------------------------------------------------------ SLNNode::SLNNode() - : FileNode( AString::GetEmpty(), Node::FLAG_ALWAYS_BUILD ) + : FileNode() { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 100; // higher default than a file node m_Type = Node::SLN_NODE; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/SettingsNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/SettingsNode.cpp index 6f33cca20..c0bdba488 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/SettingsNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/SettingsNode.cpp @@ -37,9 +37,9 @@ REFLECT_END( SettingsNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ SettingsNode::SettingsNode() -: Node( AString::GetEmpty(), Node::SETTINGS_NODE, Node::FLAG_NONE ) -, m_WorkerConnectionLimit( 15 ) -, m_DistributableJobMemoryLimitMiB( DIST_MEMORY_LIMIT_DEFAULT ) + : Node( Node::SETTINGS_NODE ) + , m_WorkerConnectionLimit( 15 ) + , m_DistributableJobMemoryLimitMiB( DIST_MEMORY_LIMIT_DEFAULT ) { // Cache path from environment Env::GetEnvVariable( "FASTBUILD_CACHE_PATH", m_CachePathFromEnvVar ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp index 696da25f7..022fc1218 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp @@ -43,7 +43,7 @@ REFLECT_END( TestNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ TestNode::TestNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_TestExecutable() , m_TestArguments() , m_TestWorkingDir() @@ -150,7 +150,7 @@ const char * TestNode::GetEnvironmentString() const Node * sn = nodeGraph.FindNode( file.m_Name ); if ( sn == nullptr ) { - sn = nodeGraph.CreateFileNode( file.m_Name ); + sn = nodeGraph.CreateNode( file.m_Name ); } else if ( sn->IsAFile() == false ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp index ba5a579a8..193e13070 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp @@ -30,7 +30,7 @@ REFLECT_END( TextFileNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ TextFileNode::TextFileNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_TextFileAlways( false ) { m_Type = TEXT_FILE_NODE; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp index fc52a1ec4..c9c873133 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp @@ -142,7 +142,7 @@ bool UnityNode::UnityFileAndOrigin::operator < ( const UnityFileAndOrigin & othe // CONSTRUCTOR //------------------------------------------------------------------------------ UnityNode::UnityNode() - : Node( AString::GetEmpty(), Node::UNITY_NODE, Node::FLAG_NONE ) + : Node( Node::UNITY_NODE ) , m_InputPathRecurse( true ) , m_InputPattern( 1, true ) , m_Files( 0, true ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VSProjectBaseNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VSProjectBaseNode.cpp index 8e60fd936..eeb32ad99 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VSProjectBaseNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VSProjectBaseNode.cpp @@ -14,8 +14,9 @@ REFLECT_END( VSProjectBaseNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ VSProjectBaseNode::VSProjectBaseNode() - : FileNode( AString::GetEmpty(), Node::FLAG_ALWAYS_BUILD ) + : FileNode() { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 100; // higher default than a file node } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp index 4d9dbf973..fc82ab480 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp @@ -90,12 +90,13 @@ REFLECT_END( XCodeProjectNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ XCodeProjectNode::XCodeProjectNode() - : FileNode( AString::GetEmpty(), Node::FLAG_ALWAYS_BUILD ) + : FileNode() , m_XCodeOrganizationName( "Organization" ) , m_XCodeBuildToolPath( "./FBuild" ) , m_XCodeBuildToolArgs( "-ide $(FASTBUILD_TARGET)" ) , m_XCodeBuildWorkingDir( "./" ) { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_Type = Node::XCODEPROJECT_NODE; ProjectGeneratorBase::GetDefaultAllowedFileExtensions( m_ProjectAllowedFileExtensions ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp index 9d1a0275c..f77408d20 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp @@ -92,8 +92,10 @@ class NodeTestHelper : public Node REFLECT_DECLARE( NodeTestHelper ) public: NodeTestHelper() - : Node( AStackString<>( "dummy" ), Node::PROXY_NODE, 0 ) - {} + : Node( Node::PROXY_NODE ) + { + SetName( AStackString<>( "placeholder" ) ); + } virtual bool Initialize( NodeGraph & /*nodeGraph*/, const BFFToken * /*funcStartIter*/, const Function * /*function*/ ) override { ASSERT( false ); @@ -169,7 +171,7 @@ void TestGraph::SingleFileNode() const TEST_ASSERT( ng.FindNode( testFileName ) == nullptr ); // create the node, and make sure we can access it by name - FileNode * node = ng.CreateFileNode( testFileName ); + FileNode * node = ng.CreateNode( testFileName ); TEST_ASSERT( ng.FindNode( testFileName ) == node ); TEST_ASSERT( fb.Build( node ) ); @@ -188,7 +190,7 @@ void TestGraph::SingleFileNodeMissing() const // make a node for a file that does not exist const AStackString<> testFileName( "ThisFileDoesNotExist.cpp" ); - FileNode * node = ng.CreateFileNode( testFileName ); + FileNode * node = ng.CreateNode( testFileName ); // make sure build still passes // a missing file is not an error. it would need to be required by something diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp index ce7c0277e..0da2ccf61 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp @@ -170,8 +170,10 @@ class BaseNode : public Node REFLECT_DECLARE( BaseNode ) public: BaseNode() - : Node( AStackString<>( "dummy" ), Node::PROXY_NODE, 0 ) - {} + : Node( Node::PROXY_NODE ) + { + SetName( AStackString<>( "placeholder" ) ); + } virtual bool Initialize( NodeGraph & /*nodeGraph*/, const BFFToken * /*funcStartIter*/, const Function * /*function*/ ) override { ASSERT( false ); From 05df79e2ebb7495ebd48e477c22c8c1891cfb1f0 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Mon, 3 Jul 2023 13:08:04 +0930 Subject: [PATCH 044/129] Code cleanup - normalize whitespace [Improvement] Database loading and various lookup performance improved slightly - use xxHash3 instead of CRC32 for node name hashes Revert "[Improvement] Database loading and various lookup performance improved slightly" This reverts commit 775c62f7153b8d3ba1243e526643557979e18ced. --- Code/.FASTBuild/HelperFunctions.bff | 2 +- Code/Core/CoreTest/Tests/TestFileIO.cpp | 4 +-- Code/Core/CoreTest/Tests/TestSemaphore.cpp | 2 +- Code/Core/Env/Assert.cpp | 6 ++--- Code/Core/Env/Env.cpp | 2 +- Code/Core/Env/WindowsHeader.h | 4 +-- Code/Core/FileIO/FileIO.cpp | 10 +++---- Code/Core/Network/Network.cpp | 2 +- Code/Core/Network/TCPConnectionPool.h | 2 +- Code/Core/Process/Process.cpp | 6 ++--- Code/Core/Process/Thread.cpp | 2 +- Code/Core/Strings/AString.cpp | 4 +-- Code/OSUI/OSWindow.cpp | 2 +- Code/Tools/FBuild/FBuild/FBuild.bff | 2 +- Code/Tools/FBuild/FBuild/Main.cpp | 2 +- .../FBuildCore/BFF/BFFBooleanExpParser.cpp | 4 +-- .../Tools/FBuild/FBuildCore/BFF/BFFParser.cpp | 4 +-- .../FBuild/FBuildCore/BFF/BFFStackFrame.cpp | 2 +- .../FBuild/FBuildCore/BFF/BFFStackFrame.h | 2 +- .../FBuildCore/BFF/Functions/FunctionIf.cpp | 4 +-- .../BFF/Functions/FunctionObjectList.cpp | 4 +-- .../BFF/Functions/FunctionObjectList.h | 2 +- .../FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp | 6 ++--- .../FBuild/FBuildCore/Cache/LightCache.cpp | 4 +-- Code/Tools/FBuild/FBuildCore/Error.cpp | 2 +- .../ExeDrivers/Compiler/CompilerDriverBase.h | 2 +- .../ExeDrivers/Compiler/CompilerDriver_CL.cpp | 4 +-- .../ExeDrivers/Compiler/CompilerDriver_CL.h | 2 +- Code/Tools/FBuild/FBuildCore/FBuild.cpp | 2 +- .../Tools/FBuild/FBuildCore/FBuildOptions.cpp | 4 +-- .../FBuild/FBuildCore/Graph/CompilerNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/Dependencies.cpp | 2 +- .../FBuild/FBuildCore/Graph/ExecNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/FileNode.cpp | 4 +-- .../FBuild/FBuildCore/Graph/LibraryNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/LinkerNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/LinkerNode.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/Node.cpp | 24 ++++++++--------- .../FBuild/FBuildCore/Graph/NodeGraph.cpp | 10 +++---- .../FBuildCore/Graph/ObjectListNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/ObjectNode.cpp | 8 +++--- .../Tools/FBuild/FBuildCore/Graph/SLNNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/TextFileNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/UnityNode.cpp | 2 +- .../Graph/VSProjectExternalNode.cpp | 4 +-- .../Helpers/ProjectGeneratorBase.cpp | 2 +- .../FBuildCore/Helpers/Report/HTMLReport.h | 2 +- .../FBuildCore/Helpers/Report/JSONReport.cpp | 4 +-- .../FBuildCore/Helpers/ToolManifest.cpp | 6 ++--- .../FBuild/FBuildCore/Helpers/ToolManifest.h | 2 +- .../Helpers/XCodeProjectGenerator.cpp | 12 ++++----- .../FBuild/FBuildCore/Protocol/Client.cpp | 10 +++---- .../FBuild/FBuildCore/Protocol/Server.cpp | 26 +++++++++---------- .../Tools/FBuild/FBuildCore/Protocol/Server.h | 2 +- .../FBuild/FBuildCore/WorkerPool/JobQueue.cpp | 2 +- .../WorkerPool/WorkerBrokerageServer.cpp | 2 +- .../RelativePaths/SubDir/subdir.bff | 2 +- .../if_file_exists_directive.bff | 2 +- .../LightCache_IncludeHierarchy/common.h | 2 +- .../LightCache_IncludeHierarchy/fbuild.bff | 2 +- .../LightCache_IncludeUsingMacro/fbuild.bff | 2 +- .../LightCache_IncludeUsingMacro2/fbuild.bff | 2 +- .../LightCache_IncludeUsingMacro2/file.1.cpp | 2 +- .../LightCache_IncludeUsingMacro2/file.2.cpp | 2 +- .../LightCache_IncludeUsingMacro3/fbuild.bff | 2 +- .../RemoteRaceSystemFailure/Fast.cpp | 2 +- .../Data/TestObjectList/Exclusions/fbuild.bff | 2 +- .../Data/TestPrecompiledHeaders/fbuild.bff | 4 +-- .../Solution_ExternalProject/fbuild.bff | 2 +- .../fbuild_WrongData.bff | 8 +++--- .../FBuildTest/Tests/TestBFFParsing.cpp | 4 +-- .../FBuildTest/Tests/TestCompressor.cpp | 2 +- .../FBuildTest/Tests/TestDependencies.cpp | 2 +- Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp | 4 +-- .../Tests/TestProjectGeneration.cpp | 2 +- .../FBuild/FBuildTest/Tests/TestUnity.cpp | 2 +- .../FBuild/FBuildWorker/FBuildWorker.bff | 8 +++--- .../FBuild/FBuildWorker/Worker/Worker.cpp | 8 +++--- .../FBuildWorker/Worker/WorkerSettings.h | 2 +- Code/fbuild.bff | 4 +-- 80 files changed, 157 insertions(+), 157 deletions(-) diff --git a/Code/.FASTBuild/HelperFunctions.bff b/Code/.FASTBuild/HelperFunctions.bff index 62102fd1d..90d6e31df 100644 --- a/Code/.FASTBuild/HelperFunctions.bff +++ b/Code/.FASTBuild/HelperFunctions.bff @@ -189,5 +189,5 @@ function CreateVCXProject_Exe( .Name Alias( '$ProjectName$-OSX-$BuildConfigName$' ) { .Targets = '$ProjectName$-Exe-OSX-$BuildConfigName$' } } #endif - + //------------------------------------------------------------------------------ diff --git a/Code/Core/CoreTest/Tests/TestFileIO.cpp b/Code/Core/CoreTest/Tests/TestFileIO.cpp index f56181451..0f3ff37bb 100644 --- a/Code/Core/CoreTest/Tests/TestFileIO.cpp +++ b/Code/Core/CoreTest/Tests/TestFileIO.cpp @@ -320,7 +320,7 @@ void TestFileIO::LongPaths() const // // Ensure long paths are correctly handled by various functions // - + #if defined( __WINDOWS__ ) // On Windows, long path support must be enabled via a registry key // Only if this is enabled can we expect our test to pass @@ -365,7 +365,7 @@ void TestFileIO::LongPaths() const filePathB.Trim( 0, 5 ); filePathB += ".copy"; TEST_ASSERT( filePathB.GetLength() == ( filePathA.GetLength() ) ); - + // long subdir 1 subDir1.Format( "%s/%s", tmpPath2.Get(), a.Get() ); TEST_ASSERT( subDir1.GetLength() == ( tmpPath2.GetLength() + 1 + 255 ) ); diff --git a/Code/Core/CoreTest/Tests/TestSemaphore.cpp b/Code/Core/CoreTest/Tests/TestSemaphore.cpp index c270ab50a..f93c09dd0 100644 --- a/Code/Core/CoreTest/Tests/TestSemaphore.cpp +++ b/Code/Core/CoreTest/Tests/TestSemaphore.cpp @@ -105,7 +105,7 @@ void TestSemaphore::WaitTimeout() const void TestSemaphore::MaxCount() const { // Only Windows supports a signall count limit for Semaphores - + // Create sempahore with a max count Semaphore s( 1 ); diff --git a/Code/Core/Env/Assert.cpp b/Code/Core/Env/Assert.cpp index e7cc80dec..fe99ec5cc 100644 --- a/Code/Core/Env/Assert.cpp +++ b/Code/Core/Env/Assert.cpp @@ -52,7 +52,7 @@ bool IsDebuggerAttached() // reason, we get a predictable result. struct kinfo_proc info; info.kp_proc.p_flag = 0; - + // Initialize mib, which tells sysctl the info we want, in this case // we're looking for information about a specific process ID. int mib[ 4 ]; @@ -60,11 +60,11 @@ bool IsDebuggerAttached() mib[ 1 ] = KERN_PROC; mib[ 2 ] = KERN_PROC_PID; mib[ 3 ] = getpid(); - + // Call sysctl size_t size = sizeof(info); VERIFY( sysctl( mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0 ) == 0 ); - + // We're being debugged if the P_TRACED flag is set. return ( ( info.kp_proc.p_flag & P_TRACED ) != 0 ); #elif defined( __LINUX__ ) diff --git a/Code/Core/Env/Env.cpp b/Code/Core/Env/Env.cpp index 275ca565b..36a6955de 100644 --- a/Code/Core/Env/Env.cpp +++ b/Code/Core/Env/Env.cpp @@ -223,7 +223,7 @@ void Env::GetExePath( AString & output ) uint32_t bufferSize = 0; VERIFY( _NSGetExecutablePath( nullptr, &bufferSize ) == -1 ); ASSERT( bufferSize > 0 ); // Updated by _NSGetExecutablePath - + // Reserve enough space (-1 since bufferSize includes the null) output.SetLength( bufferSize - 1 ); VERIFY( _NSGetExecutablePath( output.Get(), &bufferSize ) == 0 ); diff --git a/Code/Core/Env/WindowsHeader.h b/Code/Core/Env/WindowsHeader.h index 2f0a9727d..11cf8dfa8 100644 --- a/Code/Core/Env/WindowsHeader.h +++ b/Code/Core/Env/WindowsHeader.h @@ -1,7 +1,7 @@ // WindowsHeader //------------------------------------------------------------------------------ // -// Windows.h and WinSock2.h mustbe included in a specific order to avoid +// Windows.h and WinSock2.h must be included in a specific order to avoid // compile errors. This is problematic when headers include other headers // that include Windows.h, or when using Unity. // @@ -16,7 +16,7 @@ // Includes //------------------------------------------------------------------------------ -#pragma warning(push, 0) +#pragma warning(push, 0) #pragma warning(push) #pragma warning(disable:6101) // Returning uninitialized memory '*Mtu'. A successful path through the function does not set the named _Out_ parameter. #include // WinSock2.h must be first diff --git a/Code/Core/FileIO/FileIO.cpp b/Code/Core/FileIO/FileIO.cpp index 8cb5615d7..3981ca337 100644 --- a/Code/Core/FileIO/FileIO.cpp +++ b/Code/Core/FileIO/FileIO.cpp @@ -58,13 +58,13 @@ { public: typedef int (*FuncPtr)( int dirfd, const char * pathname, const struct timespec times[ 2 ], int flags ); - + OSXHelper_utimensat() { // Open the c runtime library m_LibCHandle = dlopen( "libc.dylib", RTLD_LAZY ); ASSERT( m_LibCHandle ); // This should never fail - + // See if utimensat exists m_FuncPtr = (FuncPtr)dlsym( m_LibCHandle, "utimensat" ); } @@ -677,7 +677,7 @@ } continue; // Keep processing path } - + // Path doesn't exist so keep rest of path as-is outNormalizedPath += pos; break; @@ -812,7 +812,7 @@ t[ 1 ] = t[ 0 ]; return ( (gOSXHelper_utimensat.m_FuncPtr)( 0, fileName.Get(), t, 0 ) == 0 ); } - + // Fallback to regular low-resolution filetime setting struct timeval t[ 2 ]; t[ 0 ].tv_sec = fileTime / 1000000000ULL; @@ -843,7 +843,7 @@ { return ( (gOSXHelper_utimensat.m_FuncPtr)( 0, fileName.Get(), nullptr, 0 ) == 0 ); } - + // Fallback to regular low-resolution filetime setting return ( utimes( fileName.Get(), nullptr ) == 0 ); #elif defined( __LINUX__ ) diff --git a/Code/Core/Network/Network.cpp b/Code/Core/Network/Network.cpp index 87913b648..5156455ce 100644 --- a/Code/Core/Network/Network.cpp +++ b/Code/Core/Network/Network.cpp @@ -133,7 +133,7 @@ NI_NUMERICHOST ) == 0 ); outAddresses.EmplaceBack( host ); } - + info = info->ifa_next; } diff --git a/Code/Core/Network/TCPConnectionPool.h b/Code/Core/Network/TCPConnectionPool.h index 97d99752d..a85176386 100644 --- a/Code/Core/Network/TCPConnectionPool.h +++ b/Code/Core/Network/TCPConnectionPool.h @@ -26,7 +26,7 @@ class TCPConnectionPool; // Constants //------------------------------------------------------------------------------ -namespace +namespace { static const uint32_t kDefaultConnectionTimeoutMS = ( 2 * 1000 ); static const uint32_t kDefaultSendTimeoutMS = ( 10 * 60 * 1000 ); diff --git a/Code/Core/Process/Process.cpp b/Code/Core/Process/Process.cpp index 4f315802e..d843970af 100644 --- a/Code/Core/Process/Process.cpp +++ b/Code/Core/Process/Process.cpp @@ -601,7 +601,7 @@ bool Process::ReadAllData( AString & outMem, if ( ( prevOutSize != outMem.GetLength() ) || ( prevErrSize != errMem.GetLength() ) ) { #if defined( __LINUX__ ) - // Reset sleep interval + // Reset sleep interval sleepIntervalMS = 1; #endif continue; // try reading again right away incase there is more @@ -702,7 +702,7 @@ bool Process::ReadAllData( AString & outMem, { ASSERT( false ); // error! } - + // Update length buffer.SetLength( sizeSoFar + bytesReadNow ); } @@ -749,7 +749,7 @@ bool Process::ReadAllData( AString & outMem, ASSERT( false ); // error! result = 0; // no bytes read } - + // Update length buffer.SetLength( buffer.GetLength() + (uint32_t)result ); } diff --git a/Code/Core/Process/Thread.cpp b/Code/Core/Process/Thread.cpp index 76af89591..a07a00ee7 100644 --- a/Code/Core/Process/Thread.cpp +++ b/Code/Core/Process/Thread.cpp @@ -112,7 +112,7 @@ void Thread::Start( ThreadEntryFunction func, { // Can only start if not already started ASSERT( !IsRunning() ); - + // Create structure to pass to thread ThreadStartInfo & info = *FNEW( ThreadStartInfo( func, userData, threadName ) ); MemoryBarrier(); diff --git a/Code/Core/Strings/AString.cpp b/Code/Core/Strings/AString.cpp index b2c11de05..5afb0bc99 100644 --- a/Code/Core/Strings/AString.cpp +++ b/Code/Core/Strings/AString.cpp @@ -709,7 +709,7 @@ void AString::TrimStart( char charToTrimFromStart ) uint32_t nbrCharsToRemoveFromStart = 0; const char * pos = m_Contents; const char * end = m_Contents + m_Length; - for ( ; pos < end && *pos == charToTrimFromStart; ++pos, ++nbrCharsToRemoveFromStart ) + for ( ; pos < end && *pos == charToTrimFromStart; ++pos, ++nbrCharsToRemoveFromStart ) { } @@ -723,7 +723,7 @@ void AString::TrimEnd( char charToTrimFromEnd ) uint32_t nbrCharsToRemoveFromEnd = 0; const char * pos = m_Contents + m_Length - 1; const char * end = m_Contents; - for ( ; pos >= end && *pos == charToTrimFromEnd; --pos, ++nbrCharsToRemoveFromEnd ) + for ( ; pos >= end && *pos == charToTrimFromEnd; --pos, ++nbrCharsToRemoveFromEnd ) { } diff --git a/Code/OSUI/OSWindow.cpp b/Code/OSUI/OSWindow.cpp index 677a742a3..69901c28d 100644 --- a/Code/OSUI/OSWindow.cpp +++ b/Code/OSUI/OSWindow.cpp @@ -283,7 +283,7 @@ void OSWindow::StartMessagePump() // Allow our message pump to do the work, whether it's WM_QUIT (0 bRet) or not (non-zero bRet) TranslateMessage( &msg ); - DispatchMessage( &msg ); + DispatchMessage( &msg ); } #elif defined( __OSX__ ) // This call blocks until messaged by StopMessagePump diff --git a/Code/Tools/FBuild/FBuild/FBuild.bff b/Code/Tools/FBuild/FBuild/FBuild.bff index f3c260a6c..af652c17e 100644 --- a/Code/Tools/FBuild/FBuild/FBuild.bff +++ b/Code/Tools/FBuild/FBuild/FBuild.bff @@ -106,7 +106,7 @@ } } #endif - + // Aliases //-------------------------------------------------------------------------- CreateCommonAliases( .ProjectName ) diff --git a/Code/Tools/FBuild/FBuild/Main.cpp b/Code/Tools/FBuild/FBuild/Main.cpp index 9f2cb9f22..356c1d933 100644 --- a/Code/Tools/FBuild/FBuild/Main.cpp +++ b/Code/Tools/FBuild/FBuild/Main.cpp @@ -256,7 +256,7 @@ int Main( int argc, char * argv[] ) if ( problemSavingBuildProfileJSON ) { return FBUILD_FAILED_TO_WRITE_PROFILE_JSON; - } + } return ( result == true ) ? FBUILD_OK : FBUILD_BUILD_FAILED; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFBooleanExpParser.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFBooleanExpParser.cpp index a924bcf74..9a3fb9ec8 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFBooleanExpParser.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFBooleanExpParser.cpp @@ -770,7 +770,7 @@ static bool ParseBinaryBooleanExp( const Function * function, bool lhs, BFFToken // ParseIntComparisonExp // Forms: -// bool-exp = +// bool-exp = // | number-exp ==/!=/>/>=/GetValueString().EndsWith( *pos ) ); - ASSERT( iter->GetValueString().GetLength() >= 4 ); // at least one char inside: ."x" + ASSERT( iter->GetValueString().GetLength() >= 4 ); // at least one char inside: ."x" // unescape and subsitute embedded variables AStackString<> value; @@ -222,7 +222,7 @@ bool BFFParser::Parse( BFFTokenRange & iter ) } // sanity check it is a sensible length - if ( name.GetLength() + 1 > MAX_VARIABLE_NAME_LENGTH ) // +1 for '.' will be added + if ( name.GetLength() + 1 > MAX_VARIABLE_NAME_LENGTH ) // +1 for '.' will be added { Error::Error_1014_VariableNameIsTooLong( iter, (uint32_t)value.GetLength(), (uint32_t)MAX_VARIABLE_NAME_LENGTH ); return false; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp index 539eaf478..0ce6eeadb 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp @@ -132,7 +132,7 @@ void BFFStackFrame::DisconnectStackChain() // SetVarInt //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVarInt( const AString & name, +/*static*/ void BFFStackFrame::SetVarInt( const AString & name, const BFFToken & token, int value, BFFStackFrame * frame ) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h index 73518f98d..8dcf9d15e 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h @@ -95,7 +95,7 @@ class BFFStackFrame const AString & GetLastVariableSeen() const { return m_LastVariableSeen; } BFFStackFrame * GetLastVariableSeenFrame() const { return m_LastVariableSeenFrame; } void SetLastVariableSeen( const AString & varName, BFFStackFrame * frame ) - { + { m_LastVariableSeen = varName; m_LastVariableSeenFrame = frame; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionIf.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionIf.cpp index 53c5df40b..f89555b84 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionIf.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionIf.cpp @@ -42,12 +42,12 @@ FunctionIf::FunctionIf() // Iterate the args const BFFTokenRange header( headerRange ); - bool headerResult; + bool headerResult; if ( !BFFBooleanExpParser::Parse( this, header, headerResult ) ) { return false; } - + if ( !headerResult ) { return true; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.cpp index fd9f9f312..25e9a1b4f 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.cpp @@ -229,8 +229,8 @@ bool FunctionObjectList::CheckMSVCPCHFlags_Use( const BFFToken * iter, // GetExtraOutputPaths //------------------------------------------------------------------------------ -void FunctionObjectList::GetExtraOutputPaths( const AString & args, - AString & outPDBPath, +void FunctionObjectList::GetExtraOutputPaths( const AString & args, + AString & outPDBPath, AString & outASMPath, AString & outSourceDependenciesPath ) { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.h b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.h index fc6343595..de34c9048 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.h @@ -40,7 +40,7 @@ class FunctionObjectList : public Function friend class TestObjectList; static void GetExtraOutputPaths( const AString & args, - AString & outPDBPath, + AString & outPDBPath, AString & outASMPath, AString & outSourceDependenciesPath ); static void GetExtraOutputPath( const AString * it, const AString * end, const char * option, AString & path ); diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp index 32ed62412..1010176f7 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp @@ -694,16 +694,16 @@ bool BFFTokenizer::HandleDirective_If( const BFFFile & file, const char * & pos, // Store to history operatorHistory[ numOperators++ ] = r; - + // Check for excessive complexity if ( numOperators == BFFParser::MAX_OPERATOR_HISTORY ) { Error::Error_1047_IfExpressionTooComplex( argsIter.GetCurrent() ); return false; - } + } } } - + // Apply any && operators. Any valid expression isn't going to end with an operator, so we don't check that. for ( uint32_t i = 0; i < ( numOperators - 1 ); i++ ) { diff --git a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp index 1c379ccfa..fd9fadad6 100644 --- a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp +++ b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp @@ -600,7 +600,7 @@ bool LightCache::ParseMacroName( const char * & pos, AString & outMacroName ) c = *pos; if ( ( ( c >= 'a' ) && ( c <= 'z' ) ) || ( ( c >= 'A' ) && ( c <= 'Z' ) ) || - ( ( c >= '0' ) && ( c <= '9' ) ) || + ( ( c >= '0' ) && ( c <= '9' ) ) || ( c == '_' ) ) { ++pos; @@ -868,7 +868,7 @@ const IncludedFile * LightCache::FileExists( const AString & fileName ) if ( location ) { m_IncludeDefines.Append( location->m_IncludeDefines ); - + return location; // File previously handled so we can re-use the result } } diff --git a/Code/Tools/FBuild/FBuildCore/Error.cpp b/Code/Tools/FBuild/FBuildCore/Error.cpp index 0352daa15..0fceffdf3 100644 --- a/Code/Tools/FBuild/FBuildCore/Error.cpp +++ b/Code/Tools/FBuild/FBuildCore/Error.cpp @@ -765,7 +765,7 @@ "1234567890" "~`!@#$%^&*()-_+=" "{[]}|\\" - ";:\"'" + ";:\"'" "<,.>/?"; bool printable = false; const char c = *iter->GetSourcePos(); diff --git a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriverBase.h b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriverBase.h index 9c776efbf..fda1c8549 100644 --- a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriverBase.h +++ b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriverBase.h @@ -1,4 +1,4 @@ -// CompilerDriverBase.h +// CompilerDriverBase.h //------------------------------------------------------------------------------ #pragma once diff --git a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.cpp b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.cpp index dcd56cb33..4b83f639b 100644 --- a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.cpp +++ b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.cpp @@ -143,13 +143,13 @@ CompilerDriver_CL::~CompilerDriver_CL() = default; { ++index; // Skip next arg which specifies the mode for '/sourceDependencies' return true; - } + } // Remove "/sourceDependencies" if ( IsStartOfCompilerArg_MSVC( token, "sourceDependencies" ) ) { return true; } - + return false; } diff --git a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.h b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.h index 66d0517ef..55a12af63 100644 --- a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.h +++ b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.h @@ -1,4 +1,4 @@ -// CompilerDriver_CL.h +// CompilerDriver_CL.h //------------------------------------------------------------------------------ #pragma once diff --git a/Code/Tools/FBuild/FBuildCore/FBuild.cpp b/Code/Tools/FBuild/FBuildCore/FBuild.cpp index 0ce5a1002..d602c435c 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuild.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuild.cpp @@ -804,7 +804,7 @@ bool FBuild::GenerateDotGraph( const Array< AString > & targets, const bool full OUTPUT( "Saving DOT graph file to '%s'\n", dotFileName ); // Generate - AString buffer( 10 * 1024 * 1024 ); + AString buffer( 10 * 1024 * 1024 ); m_DependencyGraph->SerializeToDotFormat( deps, fullGraph, buffer ); // Write to disk diff --git a/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp b/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp index 1491d66d4..ece79c584 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp @@ -167,7 +167,7 @@ FBuildOptions::OptionsResult FBuildOptions::ProcessCommandLine( int argc, char * } m_CacheCompressionLevel = static_cast< int16_t >( cacheCompressionLevel ); i++; // skip extra arg we've consumed - + // add to args we might pass to subprocess m_Args += ' '; m_Args += argv[ sizeIndex ]; @@ -375,7 +375,7 @@ FBuildOptions::OptionsResult FBuildOptions::ProcessCommandLine( int argc, char * thisArg.Tokenize( reportTokens, '=' ); // if there is something after the '=' sign, then we take whatever comes after as the report type - if ( reportTokens.GetSize() > 1 ) + if ( reportTokens.GetSize() > 1 ) { m_ReportType = reportTokens[ 1 ]; m_ReportType.ToLower(); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp index 9d8e41847..b97dbbe4c 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp @@ -308,7 +308,7 @@ bool CompilerNode::InitializeCompilerFamily( const BFFToken * iter, const Functi { m_CompilerFamilyEnum = CLANG_CL; return true; - } + } if ( m_CompilerFamilyString.EqualsI( "snc" ) ) { m_CompilerFamilyEnum = SNC; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Dependencies.cpp b/Code/Tools/FBuild/FBuildCore/Graph/Dependencies.cpp index d234f7f40..f291a11f8 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Dependencies.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/Dependencies.cpp @@ -56,7 +56,7 @@ void Dependencies::Load( NodeGraph & nodeGraph, ConstMemoryStream & stream ) { return; } - + SetCapacity( numDeps ); for ( uint32_t i=0; i m_LinkerAssemblyResources; bool m_LinkerLinkObjects = false; bool m_LinkerAllowResponseFile; - bool m_LinkerForceResponseFile; + bool m_LinkerForceResponseFile; AString m_LinkerStampExe; AString m_LinkerStampExeArgs; Array< AString > m_PreBuildDependencyNames; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp index 1b7ead001..ca00c8229 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp @@ -355,7 +355,7 @@ void Node::SetLastBuildTime( uint32_t ms ) // Build time uint32_t lastTimeToBuild; VERIFY( stream.Read( lastTimeToBuild ) ); - n->SetLastBuildTime( lastTimeToBuild ); + n->SetLastBuildTime( lastTimeToBuild ); // Deserialize properties Deserialize( stream, n, *n->GetReflectionInfoV() ); @@ -1050,14 +1050,14 @@ void Node::ReplaceDummyName( const AString & newName ) /*static*/ void Node::CleanMessageToPreventMSBuildFailure( const AString & msg, AString & outMsg ) { // Search for patterns that MSBuild detects and treats as errors: - // + // // : - // + // // and remove the colon so they are no longer detected: - // + // // - // - // These can be anywhere in the string, and are case and whitespace insensitive + // + // These can be anywhere in the string, and are case and whitespace insensitive const char * pos = msg.Get(); for ( ;; ) { @@ -1135,18 +1135,18 @@ bool Node::InitializePreBuildDependencies( NodeGraph & nodeGraph, const BFFToken // No - return build-wide environment return FBuild::IsValid() ? FBuild::Get().GetEnvironmentString() : nullptr; } - + // More than one caller could be retrieving the same env string // in some cases. For simplicity, we protect in all cases even // if we could avoid it as the mutex will not be heavily constested. MutexHolder mh( g_NodeEnvStringMutex ); - + // If we've previously built a custom env string, use it if ( inoutCachedEnvString ) { return inoutCachedEnvString; } - + // Caller owns the memory inoutCachedEnvString = Env::AllocEnvironmentString( envVars ); return inoutCachedEnvString; @@ -1157,14 +1157,14 @@ bool Node::InitializePreBuildDependencies( NodeGraph & nodeGraph, const BFFToken void Node::RecordStampFromBuiltFile() { m_Stamp = FileIO::GetFileLastWriteTime( m_Name ); - + // An external tool might fail to write a file. Higher level code checks for // that (see "missing despite success"), so we don't need to do anything here. if ( m_Stamp == 0 ) { return; } - + // On OS X, the 'ar' tool (for making libraries) appears to clamp the // modification time of libraries to whole seconds. On HFS/HFS+ file systems, // this doesn't matter because the resolution of the file system is 1 second. @@ -1193,7 +1193,7 @@ void Node::RecordStampFromBuiltFile() { // Set to current time FileIO::SetFileLastWriteTimeToNow( m_Name ); - + // Re-query the time from the file m_Stamp = FileIO::GetFileLastWriteTime( m_Name ); ASSERT( m_Stamp != 0 ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index 48d04c2f7..da65ef848 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -159,7 +159,7 @@ NodeGraph::~NodeGraph() corruptDBName += ".corrupt"; FileIO::FileMove( AStackString<>( nodeGraphDBFile ), corruptDBName ); // Will overwrite if needed } - + // Create a fresh DB by parsing the BFF FDELETE( oldNG ); NodeGraph * newNG = FNEW( NodeGraph ); @@ -854,7 +854,7 @@ Node * NodeGraph::CreateNode( Node::Type type, AString && name ) Node * NodeGraph::CreateNode( Node::Type type, const AString & name ) { // Where possible callers should call the move version to transfer ownership - // of strings, but calers don't always have a string to transfer so this + // of strings, but calers don't always have a string to transfer so this // helper can be called in those situations AString nameCopy; @@ -1507,8 +1507,8 @@ void NodeGraph::FindNearestNodesInternal( const AString & fullPath, Array< NodeW // // Some of these things depend on timing, so this check could conceivably run // when not stuck, but it will never falsly detect a cyclic dependency. - // - + // + // Early out if the root node is being processed if ( node->GetState() >= Node::State::BUILDING ) { @@ -1529,7 +1529,7 @@ void NodeGraph::FindNearestNodesInternal( const AString & fullPath, Array< NodeW JobQueue::Get().GetJobStats( numJobs, numJobsActive, numJobsDist, numJobsDistActive ); if ( ( numJobs > 0 ) || ( numJobsActive > 0 ) || - ( numJobsDist > 0 ) || + ( numJobsDist > 0 ) || ( numJobsDistActive > 0 ) ) { return false; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index 25565a979..4e5a381e0 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -838,7 +838,7 @@ void ObjectListNode::EnumerateInputFiles( void (*callback)( const AString & inpu { callback( file, AString::GetEmpty(), userData ); } - + // Dynamically discovered files for ( size_t i = m_ObjectListInputStartIndex; i < m_ObjectListInputEndIndex; ++i ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp index 8390e6cfe..c81c1b039 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp @@ -1411,7 +1411,7 @@ bool ObjectNode::RetrieveFromCache( Job * job ) { pchKey = xxHash3::Calc64( cacheData, cacheDataSize ); } - + const uint32_t startDecompress = uint32_t( t.GetElapsedMS() ); MultiBuffer buffer( cacheData, cacheDataSize ); @@ -2330,13 +2330,13 @@ bool ObjectNode::CompileHelper::SpawnCompiler( Job * job, { Thread::Sleep( 1 ); } - + // Add fake failure ASSERT( m_Result == 0 ); // Should not have real failures if we're faking them m_Result = 1; job->Error( "Injecting system failure (sFakeSystemFailure)\n" ); job->OnSystemError(); - + // Clear failure state sFakeSystemFailureState.Store( DISABLED ); } @@ -2551,7 +2551,7 @@ bool ObjectNode::CompileHelper::SpawnCompiler( Job * job, } } - #if !defined( __WINDOWS__) + #if !defined( __WINDOWS__) (void)stdOut; // No checks use stdOut outside of Windows right now #endif } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp index 1e57dbae0..8ab8070b8 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp @@ -164,7 +164,7 @@ SLNNode::SLNNode() { // Merge list of projects found->m_Projects.Append( folder.m_Projects ); - + // Merge list of items found->m_Items.Append( folder.m_Items ); } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp index 193e13070..25fec934b 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp @@ -57,7 +57,7 @@ TextFileNode::~TextFileNode() = default; // DetermineNeedToBuildStatic //------------------------------------------------------------------------------ /*virtual*/ bool TextFileNode::DetermineNeedToBuildStatic() const -{ +{ if ( m_TextFileAlways ) { FLOG_VERBOSE( "Need to build '%s' (TextFileAlways = true)", GetName().Get() ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp index c9c873133..6f9953ae1 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp @@ -806,7 +806,7 @@ bool UnityNode::GetIsolatedFilesFromList( Array< AString > & files ) const { return true; // No list specified so option is disabled } - + // Open file FileStream input; if ( input.Open( m_IsolateListFile.Get() ) == false ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VSProjectExternalNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VSProjectExternalNode.cpp index aa58049b4..1b03ade8c 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VSProjectExternalNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VSProjectExternalNode.cpp @@ -16,7 +16,7 @@ #if defined( __WINDOWS__ ) #include "Core/Env/WindowsHeader.h" PRAGMA_DISABLE_PUSH_CLANG( "-Wunknown-warning-option" ) - PRAGMA_DISABLE_PUSH_CLANG( "-Wreserved-identifier" ) // identifier '%s' is reserved because it starts with '_' followed by a capital letter + PRAGMA_DISABLE_PUSH_CLANG( "-Wreserved-identifier" ) // identifier '%s' is reserved because it starts with '_' followed by a capital letter PRAGMA_DISABLE_PUSH_CLANG( "-Wcast-function-type" ) // cast from '%s' (aka '%s') to '%s' (aka '%s') converts to incompatible function type PRAGMA_DISABLE_PUSH_MSVC( 4191 ) // C4191: 'reinterpret_cast': unsafe conversion from 'FARPROC' to 'Type_CleanUp' PRAGMA_DISABLE_PUSH_MSVC( 4530 ) // C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc @@ -242,7 +242,7 @@ void VSProjectExternalNode::CopyConfigs() // GetProjectTypeGuid //------------------------------------------------------------------------------ -/*virtual*/ const AString & VSProjectExternalNode::GetProjectTypeGuid() const +/*virtual*/ const AString & VSProjectExternalNode::GetProjectTypeGuid() const { return m_ProjectTypeGuid; } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ProjectGeneratorBase.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/ProjectGeneratorBase.cpp index e1616ad7a..9db7528aa 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ProjectGeneratorBase.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ProjectGeneratorBase.cpp @@ -613,7 +613,7 @@ void ProjectGeneratorBase::AddConfig( const ProjectGeneratorBaseConfig & config default: break; // Unsupported type - ignore } } - + return nullptr; } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Report/HTMLReport.h b/Code/Tools/FBuild/FBuildCore/Helpers/Report/HTMLReport.h index 8d2b4ad0b..7065f66a0 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Report/HTMLReport.h +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Report/HTMLReport.h @@ -52,7 +52,7 @@ class HTMLReport : public Report bool operator < ( const PieItem & other ) const { return m_Value > other.m_Value; } }; - + enum { DEFAULT_TABLE_WIDTH = 990 }; // Helpers diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp index c7c8fca0d..dacd9417c 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp @@ -30,7 +30,7 @@ JSONReport::~JSONReport() = default; void JSONReport::Generate( const NodeGraph & nodeGraph, const FBuildStats & stats ) { GetLibraryStats( nodeGraph, stats ); - + Write( "{\n\t" ); // build the report @@ -622,7 +622,7 @@ void JSONReport::DoIncludes() { Write( "]" ); } - else + else { Write( "\n\t]" ); } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp index f0ee99a2a..202c0c67d 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp @@ -296,7 +296,7 @@ bool ToolManifest::DeserializeFromRemote( IOStream & ms ) if ( !ms.Read( toolId ) || !ms.Read( mainExecutablePath ) || !ms.Read( numFiles ) || - ( AString::StrLen( mainExecutablePath.Get() ) != mainExecutablePath.GetLength() ) || + ( AString::StrLen( mainExecutablePath.Get() ) != mainExecutablePath.GetLength() ) || ( numFiles == 0 ) || // Must have at least 1 file ( toolId != m_ToolId ) ) // Must have correct toolId { @@ -607,7 +607,7 @@ bool ToolManifest::ReceiveFileData( uint32_t fileId, // Any replacement packet integrity validation should be not specific to // these packets and belongs at a higher level. static_assert( Protocol::PROTOCOL_VERSION_MAJOR == 22, "Remove backwards compat shims" ); - + // When running tests we should be using latest protocols which don't // have the bug anymore so this should never happen ASSERT( false && "Corrupt file data" ); // Catch errors during development @@ -701,7 +701,7 @@ bool ToolManifest::ReceiveFileData( uint32_t fileId, // Get path to file AStackString<> fileName; GetRemoteFilePath( fileId, fileName ); - + // Make modification time now FileIO::SetFileLastWriteTimeToNow( fileName ); } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.h b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.h index efa78756e..05fc34041 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.h +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.h @@ -115,7 +115,7 @@ class ToolManifest : public Struct const char * GetRemoteEnvironmentString() const { return m_RemoteEnvironmentString; } static void GetRelativePath( const AString & root, const AString & otherFile, AString & otherFileRelativePath ); - + #if defined( __OSX__ ) || defined( __LINUX__ ) void TouchFiles() const; #endif diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/XCodeProjectGenerator.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/XCodeProjectGenerator.cpp index 118a85795..b6bd187d6 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/XCodeProjectGenerator.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/XCodeProjectGenerator.cpp @@ -83,7 +83,7 @@ const AString & XCodeProjectGenerator::GenerateUserSchemeMangementPList() "\t\t\tprimary\n" "\t\t\t\n" "\t\t\n" - "\t\n", + "\t\n", pbxNativeTargetGUID.Get() ); // Footer @@ -147,7 +147,7 @@ const AString & XCodeProjectGenerator::GenerateXCScheme() " \n" " \n" " \n", - pbxLegacyTargetGUID.Get(), m_ProjectName.Get(), m_ProjectName.Get(), m_ProjectName.Get() ); + pbxLegacyTargetGUID.Get(), m_ProjectName.Get(), m_ProjectName.Get(), m_ProjectName.Get() ); // Test Action m_Tmp.AppendFormat( " \n" " \n", - escapedArgument.Get() ); - } + escapedArgument.Get() ); + } m_Tmp.AppendFormat( " \n" ); } m_Tmp.AppendFormat( " \n" @@ -397,7 +397,7 @@ void XCodeProjectGenerator::WriteFolders() "\t\t\tchildren = (\n", pbxGroupGUID.Get() ); } - + // Child Files for ( const File * file : folder->m_Files ) { @@ -813,7 +813,7 @@ void XCodeProjectGenerator::WriteString( uint32_t indentDepth, { tabs += '\t'; } - + // Empty strings and strings with spaces are quoted const char quoteString = ShouldQuoteString( value ); const char * const formatString = quoteString ? "%s%s = \"%s\";\n" diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp index 8a5a3e971..85d9a2681 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp @@ -479,14 +479,14 @@ void Client::Process( const ConnectionInfo * connection, const Protocol::MsgRequ // If we will write the results to the cache, and this node is cacheable // then we want to respect higher cache compression levels if set const int16_t cacheCompressionLevel = FBuild::Get().GetOptions().m_CacheCompressionLevel; - if ( ( cacheCompressionLevel != 0 ) && - ( FBuild::Get().GetOptions().m_UseCacheWrite ) && + if ( ( cacheCompressionLevel != 0 ) && + ( FBuild::Get().GetOptions().m_UseCacheWrite ) && ( job->GetNode()->CastTo< ObjectNode >()->ShouldUseCache() ) ) { resultCompressionLevel = Math::Max( resultCompressionLevel, cacheCompressionLevel ); } } - + // Take note of the results compression level so we know to expect // compressed results job->SetResultCompressionLevel( resultCompressionLevel ); @@ -548,7 +548,7 @@ void Client::ProcessJobResultCommon( const ConnectionInfo * connection, bool isC uint32_t buildTime; ms.Read( buildTime ); - + uint16_t remoteThreadId = 0; ms.Read( remoteThreadId ); @@ -694,7 +694,7 @@ void Client::ProcessJobResultCommon( const ConnectionInfo * connection, bool isC if ( result == true ) { // built ok - serialize to disc - + ObjectNode * objectNode = node->CastTo< ObjectNode >(); // Store to cache if needed diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index cf5f0db68..aacc2bfba 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -163,13 +163,13 @@ bool Server::IsSynchingTool( AString & statusStr ) const // This is usually null here, but might need to be freed if // we had the connection drop between message and payload FREE( (void *)( cs->m_CurrentMessage ) ); - + // delete any jobs where we were waiting on Tool synchronization for ( Job * job : cs->m_WaitingJobs ) { delete job; } - + FDELETE cs; } } @@ -331,11 +331,11 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgJob // deserialize job ConstMemoryStream ms( payload, payloadSize ); - + Job * job = FNEW( Job( ms ) ); job->SetUserData( cs ); job->SetResultCompressionLevel( msg->GetResultCompressionLevel() ); - + // Get ToolId const uint64_t toolId = msg->GetToolId(); ASSERT( toolId ); @@ -370,8 +370,8 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgJob else { // Take ownership of toolchain - manifest->SetUserData( (void *)connection ); - + manifest->SetUserData( (void *)connection ); + const bool hasManifest = ( manifest->GetFiles().IsEmpty() == false ); if ( hasManifest ) { @@ -382,7 +382,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgJob { // Manifest was not sync'd. This can happen if disconnection // occurs before the manifest was received. - + // request manifest const Protocol::MsgRequestManifest reqMsg( toolId ); reqMsg.Send( connection ); @@ -435,7 +435,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgMani // when dealing with backwards compatibility with old workers) // If we ever break protocol compatibility, we can remove special handling static_assert( Protocol::PROTOCOL_VERSION_MAJOR == 22, "Remove backwards compat shims" ); - + // This should not happen with latest code so we want to catch that when // debugging ASSERT( false && "MsgManifest corrupt" ); @@ -496,7 +496,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgFile // when dealing with backwards compatibility with old workers) // If we ever break protocol compatibility, we can remove special handling static_assert( Protocol::PROTOCOL_VERSION_MAJOR == 22, "Remove backwards compat shims" ); - + // This should not happen with latest code so we want to catch that when // debugging ASSERT( false && "MsgFile corrupt" ); @@ -596,7 +596,7 @@ void Server::ThreadFunc() FinalizeCompletedJobs(); FindNeedyClients(); - + TouchToolchains(); JobQueueRemote::Get().MainThreadWait( 100 ); @@ -613,7 +613,7 @@ void Server::FindNeedyClients() } PROFILE_FUNCTION; - + // determine job availability int32_t availableJobs = (int32_t)WorkerThreadRemote::GetNumCPUsToUse(); if ( availableJobs == 0 ) @@ -727,9 +727,9 @@ void Server::FinalizeCompletedJobs() { ASSERT( cs->m_NumJobsActive.Load() > 0 ); cs->m_NumJobsActive.Decrement(); - + MutexHolder mh2( cs->m_Mutex ); - + if ( job->GetResultCompressionLevel() == 0 ) { // Uncompressed diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h index 8be313ed8..8b0643e0c 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h @@ -94,7 +94,7 @@ class Server : public TCPConnectionPool mutable Mutex m_ToolManifestsMutex; Array< ToolManifest * > m_Tools; - + #if defined( __OSX__ ) || defined( __LINUX__ ) Timer m_TouchToolchainTimer; #endif diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp index 1a868dd81..914c669ee 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp @@ -797,7 +797,7 @@ void JobQueue::FinishedProcessingJob( Job * job, bool success, bool wasARemoteJo { // nothing to check } - else + else { // build completed ok, or retrieved from cache... ASSERT( ( result == Node::NODE_RESULT_OK ) || ( result == Node::NODE_RESULT_OK_CACHE ) ); diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerBrokerageServer.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerBrokerageServer.cpp index 2eb582244..c5e74f31b 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerBrokerageServer.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerBrokerageServer.cpp @@ -66,7 +66,7 @@ void WorkerBrokerageServer::SetAvailability( bool available ) const float elapsedTime = m_TimerLastUpdate.GetElapsed(); if ( elapsedTime >= sBrokerageAvailabilityUpdateTime ) { - // If settings have changed, (re)create the file + // If settings have changed, (re)create the file // If settings have not changed, update the modification timestamp const WorkerSettings & workerSettings = WorkerSettings::Get(); const uint64_t settingsWriteTime = workerSettings.GetSettingsWriteTime(); diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/SubDir/subdir.bff b/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/SubDir/subdir.bff index c604bc41a..4a34252ec 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/SubDir/subdir.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/SubDir/subdir.bff @@ -11,4 +11,4 @@ Print( 'OK-2B' ) // Should not see file in root (one level up) #if !file_exists( "if_file_exists_directive.bff" ) Print( 'OK-4' ) -#endif \ No newline at end of file +#endif diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/if_file_exists_directive.bff b/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/if_file_exists_directive.bff index 49c8e0aeb..6c44fe449 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/if_file_exists_directive.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/if_file_exists_directive.bff @@ -6,4 +6,4 @@ Print("File exists") #else Print("File does not exist") -#endif \ No newline at end of file +#endif diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/common.h b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/common.h index 4314c228d..3e2f55090 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/common.h +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/common.h @@ -1 +1 @@ -#include "file.h" \ No newline at end of file +#include "file.h" diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/fbuild.bff index 4734f4222..029cc586a 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/fbuild.bff @@ -11,7 +11,7 @@ Settings {} // use Standard Environment ObjectList( 'ObjectList' ) { - .CompilerInputFiles = { + .CompilerInputFiles = { '$TestRoot$/Data/TestCache/LightCache_IncludeHierarchy/Folder1/file.cpp' '$TestRoot$/Data/TestCache/LightCache_IncludeHierarchy/Folder2/file.cpp' } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro/fbuild.bff index f05637663..9b560117b 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro/fbuild.bff @@ -10,7 +10,7 @@ Settings {} // use Standard Environment ObjectList( 'ObjectList' ) { - .CompilerInputFiles = { + .CompilerInputFiles = { '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro/file.1.cpp' '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro/file.2.cpp' } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/fbuild.bff index 292dab4bb..66d88d859 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/fbuild.bff @@ -10,7 +10,7 @@ Settings {} // use Standard Environment ObjectList( 'ObjectList' ) { - .CompilerInputFiles = { + .CompilerInputFiles = { '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp' '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp' } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp index e9f650537..1837eb084 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp @@ -3,4 +3,4 @@ #include "header1.h" // now we include via the macro -#include INCLUDE_VIA_MACRO +#include INCLUDE_VIA_MACRO diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp index e9f650537..1837eb084 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp @@ -3,4 +3,4 @@ #include "header1.h" // now we include via the macro -#include INCLUDE_VIA_MACRO +#include INCLUDE_VIA_MACRO diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro3/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro3/fbuild.bff index 6dbb75d8b..0b6adca9d 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro3/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro3/fbuild.bff @@ -10,7 +10,7 @@ Settings {} // use Standard Environment ObjectList( 'ObjectList' ) { - .CompilerInputFiles = { + .CompilerInputFiles = { '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro3/file.cpp' } .CompilerOutputPath = '$Out$/Test/Cache/LightCache_IncludeUsingMacro3/' diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestDistributed/RemoteRaceSystemFailure/Fast.cpp b/Code/Tools/FBuild/FBuildTest/Data/TestDistributed/RemoteRaceSystemFailure/Fast.cpp index 07a18511a..402388caa 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestDistributed/RemoteRaceSystemFailure/Fast.cpp +++ b/Code/Tools/FBuild/FBuildTest/Data/TestDistributed/RemoteRaceSystemFailure/Fast.cpp @@ -1 +1 @@ -// No code needed - test functions with an empty object file \ No newline at end of file +// No code needed - test functions with an empty object file diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/Exclusions/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/Exclusions/fbuild.bff index 4358796d3..cdc8803ac 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/Exclusions/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/Exclusions/fbuild.bff @@ -87,4 +87,4 @@ Alias( 'Test' ) 'ExcludePattern-ForwardSlash' 'ExcludePattern-Backslash' } -} \ No newline at end of file +} diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff index 9224da58d..e71150069 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff @@ -64,10 +64,10 @@ Executable( "PCHTest" ) ObjectList( "PCHTestClang-Windows" ) { Using( .ToolChain_Clang_Windows ) - + .PCHInputFile = "Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/PrecompiledHeader.cpp" .PCHOutputFile = "$Out$\Test\PrecompiledHeaders\Clang-Windows\PrecompiledHeader.pch" - + .CompilerOptions + ' /Yu"PrecompiledHeader.h" /Fp"$PCHOutputFile$"' + ' "/ITools/FBuild/FBuildTest/Data/TestPrecompiledHeaders"' + ' -Wno-unused-macros' diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild.bff index bf116db50..e7480aedd 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild.bff @@ -54,7 +54,7 @@ VSSolution( 'ExternalProjectSolution' ) // only uncomment this if you have VSProjTypeExtractor / Visual Studio with C# support available (which is not likely the case on CI) ;.SolutionProjects + { 'External', 'ExternalMissingBraces' } - + // only uncomment this if you have VSProjTypeExtractor / Visual Studio with Python support available (which is not likely the case on CI) ;.SolutionProjects + { 'ExternalDummyProject_2' } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild_WrongData.bff b/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild_WrongData.bff index 4d85fd3df..20e34f4a0 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild_WrongData.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild_WrongData.bff @@ -24,10 +24,10 @@ VSSolution( 'ExternalWrongDataProjectSolution' ) { .SolutionProjects = { 'Proj1w', 'Proj2w' } - // this will always fail, even if you have VSProjTypeExtractor / Visual Studio with WiX toolset support available (which is not likely the case on CI) - // and should do so, that's what this test validates - .SolutionProjects + { 'ExternalDummyProject_3' } - + // this will always fail, even if you have VSProjTypeExtractor / Visual Studio with WiX toolset support available (which is not likely the case on CI) + // and should do so, that's what this test validates + .SolutionProjects + { 'ExternalDummyProject_3' } + .SolutionX64Debug = [ .Platform = 'x64' .Config = 'Debug' ] .SolutionX64Release = [ .Platform = 'x64' .Config = 'Release' .SolutionDeployProjects = 'Proj1w' ] .SolutionConfigs = { .SolutionX64Debug, .SolutionX64Release } diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp index 6794d743f..2d79d76e0 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp @@ -115,7 +115,7 @@ REGISTER_TESTS_BEGIN( TestBFFParsing ) REGISTER_TEST( IfExistsDirective ) REGISTER_TEST( IfFileExistsDirective ) REGISTER_TEST( IfFileExistsDirective_RelativePaths ) - REGISTER_TEST( IfBooleanOperators ) + REGISTER_TEST( IfBooleanOperators ) REGISTER_TEST( ElseDirective ) REGISTER_TEST( ElseDirective_Bad ) REGISTER_TEST( ElseDirective_Bad2 ) @@ -543,7 +543,7 @@ void TestBFFParsing::IfFileExistsDirective_RelativePaths() const { // file_exists treats paths the same way as #include // (paths are relative to the bff) - + FBuildTestOptions options; options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/root.bff"; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp index 41e1e5bf6..941b6b703 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp @@ -142,7 +142,7 @@ void TestCompressor::CompressHelper( const char * fileName ) const // Compress at various compression levels const int32_t compressionLevels[] = - { + { 0, // Disabled -256, -128, -64, -32, -16, -8, -4, -2, -1, // LZ4 1, 3, 6, 9, 12 // LZ4 HC diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestDependencies.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestDependencies.cpp index c4ec59e9f..af4f7201d 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestDependencies.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestDependencies.cpp @@ -64,7 +64,7 @@ void TestDependencies::Add() const // Node with defaults { Dependencies d; - d.Add( nodes[ 0 ] ); + d.Add( nodes[ 0 ] ); TEST_ASSERT( d.IsEmpty() == false ); TEST_ASSERT( d.GetSize() == 1 ); TEST_ASSERT( d.GetCapacity() > 0 ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp index e06606f0d..3033cf3d4 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp @@ -130,7 +130,7 @@ void TestIf::IfFunctionBool() const // Unary TEST_EXP_TRUE( "", "!false" ); TEST_EXP_FALSE( "", "!true" ); - + // Binary TEST_EXP_TRUE( "", "false != true" ); @@ -182,7 +182,7 @@ void TestIf::IfFunctionInt() const TEST_EXP_TRUE( "", "2 < 3" ); TEST_EXP_TRUE( "", "2 <= 3" ); TEST_EXP_FALSE( "", "2 <= 1" ); - + // Greater Than TEST_EXP_TRUE( "", "2 > 1" ); TEST_EXP_TRUE( "", "2 >= 1" ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp index f58fba393..a86bfd574 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp @@ -1393,7 +1393,7 @@ void TestProjectGeneration::VSExternalProj_ExternalProject() const CheckStatsNode( 1, 1, Node::ALIAS_NODE ); // because of the external module, peek how many of them were actually processed, depending if using the module is - // enforced or not in the actual fbuild.bff + // enforced or not in the actual fbuild.bff const FBuildStats& stats = FBuild::Get().GetStats(); const FBuildStats::Stats& nodeStatsExternal = stats.GetStatsFor( Node::VSPROJEXTERNAL_NODE ); const size_t actualNumExtSeen = nodeStatsExternal.m_NumProcessed; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp index 097dab02e..0ffa693ff 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp @@ -195,7 +195,7 @@ void TestUnity::DetectDeletedUnityFiles() const { // Ensure that a generated Unity file that has been deleted is // detected and regenerated - + EnsureFileDoesNotExist( "../tmp/Test/Unity/Unity1.cpp" ); EnsureFileDoesNotExist( "../tmp/Test/Unity/Unity2.cpp" ); diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff index d7e37a276..461c31fd4 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff @@ -55,7 +55,7 @@ // Output .CompilerOutputPath = '$OutputBase$/$ProjectPath$/' - + #if __WINDOWS__ .CompilerOptions + ' -D"PSAPI_VERSION=1"' #endif @@ -85,7 +85,7 @@ + 'FBuildWorker-Res-$Platform$-$BuildConfigName$' #endif #if __OSX__ - + '$ProjectName$-OSXRes-$Platform$-$BuildConfigName$' + + '$ProjectName$-OSXRes-$Platform$-$BuildConfigName$' #endif #if __LINUX__ .LinkerOutput = '$OutputBase$/$ProjectPath$/fbuildworker$ExeExtension$' // NOTE: lower case @@ -135,7 +135,7 @@ ^ProjectConfigs + .ProjectConfig #endif } - + // Create Universal binaries #if CLANG_SUPPORTS_ARMOSX ForEach( .BuildConfig in .BuildConfigs ) @@ -148,7 +148,7 @@ } } #endif - + // Aliases //-------------------------------------------------------------------------- CreateCommonAliases( .ProjectName ) diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp index 3aaf8cfc2..586a820c3 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp @@ -278,17 +278,17 @@ bool Worker::HasEnoughMemory() return ( m_LastMemoryCheckResult != 0 ); } m_TimerLastMemoryCheck.Start(); - + PERFORMANCE_INFORMATION memInfo; memInfo.cb = sizeof( memInfo ); if ( GetPerformanceInfo( &memInfo, sizeof( memInfo ) ) ) { const uint64_t limitMemSize = memInfo.CommitLimit * memInfo.PageSize; const uint64_t currentMemSize = memInfo.CommitTotal * memInfo.PageSize; - + // Calculate the free memory in MiB. const uint64_t freeMemSize = ( limitMemSize - currentMemSize ) / MEGABYTE; - + // Check if the free memory is high enough const WorkerSettings & ws = WorkerSettings::Get(); if ( freeMemSize > ws.GetMinimumFreeMemoryMiB() ) @@ -297,7 +297,7 @@ bool Worker::HasEnoughMemory() return true; } } - + // The machine doesn't have enough memory or query failed. Exclude this machine from worker pool. m_LastMemoryCheckResult = 0; return false; diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h b/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h index c8a02cfbc..cb1ae6a2f 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h +++ b/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h @@ -50,7 +50,7 @@ class WorkerSettings : public Singleton< WorkerSettings > private: Mode m_Mode; - uint32_t m_IdleThresholdPercent; + uint32_t m_IdleThresholdPercent; uint32_t m_NumCPUsToUse; bool m_StartMinimized; uint64_t m_SettingsWriteTime; // FileTime of settings when last changed/written to disk diff --git a/Code/fbuild.bff b/Code/fbuild.bff index d29eef84e..9b6e2bb4a 100644 --- a/Code/fbuild.bff +++ b/Code/fbuild.bff @@ -425,7 +425,7 @@ ForEach( .BuildConfig in .BuildConfigs ) Using( .BuildConfig ) Alias( 'All-$Platform$-$BuildConfigName$' ) { .Targets = .'Targets_$Platform$_$BuildConfigName$' } - + // Create additional Universal targets #if CLANG_SUPPORTS_ARMOSX If( .Platform == 'ARMOSX' ) @@ -603,7 +603,7 @@ Alias( 'All+Tests' ) ForEach( .BuildConfig in .BuildConfigs ) { Using( .BuildConfig ) - .ProjectConfig = [ + .ProjectConfig = [ Using( .'Project_$Platform$_$BuildConfigName$' ) .ProjectBuildCommand = 'cd ^$(SolutionDir)\..\..\Code\ & fbuild solution -vs' .ProjectRebuildCommand = 'cd ^$(SolutionDir)\..\..\Code\ & fbuild solution -vs -clean' From 1063d74c3674ea47369e47024c22b58c2cfcd5f9 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Mon, 3 Jul 2023 17:37:47 +0930 Subject: [PATCH 045/129] [Improvement] Database loading and various lookup performance improved slightly - use xxHash3 instead of CRC32 for node name hashes --- Code/Tools/FBuild/FBuildCore/Graph/Node.cpp | 14 ++++++++++++-- Code/Tools/FBuild/FBuildCore/Graph/Node.h | 6 ++++-- Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp | 11 +++++------ .../FBuild/FBuildCore/Helpers/Report/Report.cpp | 4 ++-- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp index ca00c8229..35f467c97 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp @@ -47,7 +47,7 @@ #include "Core/FileIO/FileIO.h" #include "Core/FileIO/IOStream.h" #include "Core/FileIO/PathUtils.h" -#include "Core/Math/CRC32.h" +#include "Core/Math/xxHash.h" #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" #include "Core/Profile/Profile.h" @@ -715,7 +715,7 @@ void Node::SetLastBuildTime( uint32_t ms ) //------------------------------------------------------------------------------ void Node::SetName( AString && name ) { - m_NameCRC = CRC32::CalcLower( name ); + m_NameHash = CalcNameHash( name ); m_Name = Move( name ); } @@ -1045,6 +1045,16 @@ void Node::ReplaceDummyName( const AString & newName ) NodeGraph::CleanPath( path, outFixedPath ); } +// CalcNameHash +//------------------------------------------------------------------------------ +/*static*/ uint32_t Node::CalcNameHash( const AString & name ) +{ + // xxHash3 returns a 64 bit hash and we use the lower 32 bits + AStackString<> nameLower( name ); + nameLower.ToLower(); + return static_cast( xxHash3::Calc64( nameLower ) ); +} + // CleanMessageToPreventMSBuildFailure //------------------------------------------------------------------------------ /*static*/ void Node::CleanMessageToPreventMSBuildFailure( const AString & msg, AString & outMsg ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.h b/Code/Tools/FBuild/FBuildCore/Graph/Node.h index be53125f8..e7ec0a4ab 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.h @@ -127,7 +127,7 @@ class Node : public Struct virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * funcStartIter, const Function * function ) = 0; virtual ~Node(); - inline uint32_t GetNameCRC() const { return m_NameCRC; } + uint32_t GetNameHash() const { return m_NameHash; } inline Type GetType() const { return m_Type; } inline const char * GetTypeName() const { return s_NodeTypeNames[ m_Type ]; } inline static const char * GetTypeName( Type t ) { return s_NodeTypeNames[ t ]; } @@ -181,6 +181,8 @@ class Node : public Struct inline const Dependencies & GetStaticDependencies() const { return m_StaticDependencies; } inline const Dependencies & GetDynamicDependencies() const { return m_DynamicDependencies; } + static uint32_t CalcNameHash( const AString & name ); + static void CleanMessageToPreventMSBuildFailure( const AString & msg, AString & outMsg ); protected: @@ -256,7 +258,7 @@ class Node : public Struct // Note: Unused 2 bytes here uint32_t m_RecursiveCost = 0; // Recursive cost used during task ordering Node * m_Next = nullptr; // Node map in-place linked list pointer - uint32_t m_NameCRC; // Hash of mName. **Set by constructor** + uint32_t m_NameHash; // Hash of mName uint32_t m_LastBuildTimeMs = 0; // Time it took to do last known full build of this node uint32_t m_ProcessingTime = 0; // Time spent on this node during this build uint32_t m_CachingTime = 0; // Time spent caching this node diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index da65ef848..b35d92859 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -45,7 +45,6 @@ #include "Core/FileIO/FileStream.h" #include "Core/FileIO/MemoryStream.h" #include "Core/FileIO/PathUtils.h" -#include "Core/Math/CRC32.h" #include "Core/Math/xxHash.h" #include "Core/Mem/Mem.h" #include "Core/Process/Thread.h" @@ -887,7 +886,7 @@ void NodeGraph::AddNode( Node * node ) ASSERT( FindNodeInternal( node->GetName() ) == nullptr ); // node name must be unique // track in NodeMap - const uint32_t crc = CRC32::CalcLower( node->GetName() ); + const uint32_t crc = Node::CalcNameHash( node->GetName() ); const size_t key = ( crc & m_NodeMapMaxKey ); node->m_Next = m_NodeMap[ key ]; m_NodeMap[ key ] = node; @@ -1318,15 +1317,15 @@ Node * NodeGraph::FindNodeInternal( const AString & fullPath ) const { ASSERT( Thread::IsMainThread() ); - const uint32_t crc = CRC32::CalcLower( fullPath ); - const size_t key = ( crc & m_NodeMapMaxKey ); + const uint32_t hash = Node::CalcNameHash( fullPath ); + const size_t key = ( hash & m_NodeMapMaxKey ); Node * n = m_NodeMap[ key ]; while ( n ) { - if ( n->GetNameCRC() == crc ) + if ( n->GetNameHash() == hash ) { - if ( n->GetName().CompareI( fullPath ) == 0 ) + if ( n->GetName().EqualsI( fullPath ) ) { return n; } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Report/Report.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Report/Report.cpp index c0437e49a..53fe4093b 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Report/Report.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Report/Report.cpp @@ -307,7 +307,7 @@ Report::IncludeStatsMap::~IncludeStatsMap() Report::IncludeStats * Report::IncludeStatsMap::Find( const Node * node ) const { // caculate table entry - const uint32_t hash = node->GetNameCRC(); + const uint32_t hash = node->GetNameHash(); const uint32_t key = ( hash & 0xFFFF ); IncludeStats * item = m_Table[ key ]; @@ -330,7 +330,7 @@ Report::IncludeStats * Report::IncludeStatsMap::Find( const Node * node ) const Report::IncludeStats * Report::IncludeStatsMap::Insert( const Node * node ) { // caculate table entry - const uint32_t hash = node->GetNameCRC(); + const uint32_t hash = node->GetNameHash(); const uint32_t key = ( hash & 0xFFFF ); // insert new item From 979f6054e557d1628f62b8f532614cb027dec668 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 4 Jul 2023 11:26:59 +0930 Subject: [PATCH 046/129] [Improvement] Reduce cost of queueing jobs during graph traversal - avoid re-sorting a list when we can merge sorted lists --- .../FBuild/FBuildCore/WorkerPool/JobQueue.cpp | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp index 914c669ee..d29668c73 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp @@ -34,7 +34,6 @@ class JobCostSorter //------------------------------------------------------------------------------ JobSubQueue::JobSubQueue() : m_Count( 0 ) - , m_Jobs( 1024, true ) { } @@ -57,6 +56,8 @@ uint32_t JobSubQueue::GetCount() const //------------------------------------------------------------------------------ void JobSubQueue::QueueJobs( Array< Node * > & nodes ) { + PROFILE_FUNCTION; + // Create wrapper Jobs around Nodes Array< Job * > jobs( nodes.GetSize() ); for ( Node * node : nodes ) @@ -73,16 +74,43 @@ void JobSubQueue::QueueJobs( Array< Node * > & nodes ) MutexHolder mh( m_Mutex ); const bool wasEmpty = m_Jobs.IsEmpty(); - m_Jobs.Append( jobs ); AtomicAdd( &m_Count, (uint32_t)jobs.GetSize() ); if ( wasEmpty ) { + m_Jobs.Swap( jobs ); return; // skip re-sorting } - // sort merged lists - m_Jobs.Sort( sorter ); + // Merge lists + Array< Job * > mergedList; + mergedList.SetSize( m_Jobs.GetSize() + jobs.GetSize() ); + Job ** dst = mergedList.Begin(); + Job ** src1 = m_Jobs.Begin(); + const Job * const * end1 = m_Jobs.End(); + Job ** src2 = jobs.Begin(); + const Job * const * end2 = jobs.End(); + while ( ( src1 < end1 ) && ( src2 < end2 ) ) + { + if ( sorter( *src1, *src2 ) ) + { + *dst++ = *src1++; + } + else + { + *dst++ = *src2++; + } + } + while ( src1 < end1 ) + { + *dst++ = *src1++; + } + while ( src2 < end2 ) + { + *dst++ = *src2++; + } + ASSERT( dst == mergedList.End() ); + m_Jobs.Swap( mergedList ); } // RemoveJob From f10f514098a84a8f7a2dd3031d78980aeea4a04a Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 4 Jul 2023 14:42:37 +0930 Subject: [PATCH 047/129] Remove unnecessary atomic op generating Job Ids - these are always created on the main thread --- Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp index e1f0ff13f..5ffa3661e 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp @@ -27,7 +27,9 @@ static uint32_t s_LastJobId( 0 ); Job::Job( Node * node ) : m_Node( node ) { - m_JobId = AtomicInc( &s_LastJobId ); + // Constructor that assigns JobId can only be called on the main thread. + ASSERT( Thread::IsMainThread() ); + m_JobId = ++s_LastJobId; } // CONSTRUCTOR From 6aafde9b5723e4c974da035c87ab01a54c0f78dc Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 4 Jul 2023 15:00:22 +0930 Subject: [PATCH 048/129] Remove unused forceClean param from DoDynamicDependencies --- Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/CSNode.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp | 4 +--- Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp | 4 ++-- Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/Node.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/Node.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp | 8 +++----- Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h | 4 ++-- Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/TestNode.h | 2 +- 15 files changed, 19 insertions(+), 23 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp index 12a220a3d..9e15eb50c 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp @@ -127,7 +127,7 @@ CSNode::~CSNode() = default; // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool CSNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool /*forceClean*/ ) +/*virtual*/ bool CSNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { // clear dynamic deps from previous passes m_DynamicDependencies.Clear(); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.h b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.h index 5af00a494..b043e3ad2 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.h @@ -25,7 +25,7 @@ class CSNode : public FileNode static inline Node::Type GetTypeS() { return Node::CS_NODE; } private: - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; CompilerNode * GetCompiler() const; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp index 162af65ec..e2fa8ca2e 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp @@ -82,10 +82,8 @@ CopyDirNode::~CopyDirNode() = default; // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool CopyDirNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) +/*virtual*/ bool CopyDirNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { - (void)forceClean; // dynamic deps are always re-added here, so this is meaningless - m_DynamicDependencies.Clear(); ASSERT( !m_StaticDependencies.IsEmpty() ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h index ce56ad904..96c752d65 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h @@ -25,7 +25,7 @@ class CopyDirNode : public Node virtual bool IsAFile() const override; private: - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; // Exposed Properties diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp index c2cb1e3ba..3388cd6e5 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp @@ -121,7 +121,7 @@ ExecNode::~ExecNode() // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool ExecNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool /*forceClean*/ ) +/*virtual*/ bool ExecNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { // clear dynamic deps from previous passes m_DynamicDependencies.Clear(); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h index 249861869..d2dc14064 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h @@ -23,7 +23,7 @@ class ExecNode : public FileNode static inline Node::Type GetTypeS() { return Node::EXEC_NODE; } private: - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual bool DetermineNeedToBuildStatic() const override; virtual BuildResult DoBuild( Job * job ) override; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp index bdf3b5863..0a5ccbe54 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp @@ -121,9 +121,9 @@ LibraryNode::~LibraryNode() // GatherDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool LibraryNode::GatherDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) +/*virtual*/ bool LibraryNode::GatherDynamicDependencies( NodeGraph & nodeGraph ) { - if ( ObjectListNode::GatherDynamicDependencies( nodeGraph, forceClean ) == false ) + if ( ObjectListNode::GatherDynamicDependencies( nodeGraph ) == false ) { return false; // GatherDynamicDependencies will have emited an error } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h index 86eff991e..f33b164ba 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h @@ -42,7 +42,7 @@ class LibraryNode : public ObjectListNode private: friend class FunctionLibrary; - virtual bool GatherDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool GatherDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; // internal helpers diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp index 35f467c97..caee10dc1 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp @@ -136,7 +136,7 @@ Node::~Node() = default; // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool Node::DoDynamicDependencies( NodeGraph &, bool ) +/*virtual*/ bool Node::DoDynamicDependencies( NodeGraph & /*nodeGraph*/ ) { return true; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.h b/Code/Tools/FBuild/FBuildCore/Graph/Node.h index e7ec0a4ab..f7d788ca8 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.h @@ -211,7 +211,7 @@ class Node : public Struct // each node implements a subset of these as needed virtual bool DetermineNeedToBuildStatic() const; virtual bool DetermineNeedToBuildDynamic() const; - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ); + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ); virtual BuildResult DoBuild( Job * job ); virtual BuildResult DoBuild2( Job * job, bool racingRemoteJob ); virtual bool Finalize( NodeGraph & nodeGraph ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index b35d92859..9f97b673d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -1018,7 +1018,7 @@ void NodeGraph::BuildRecurse( Node * nodeToBuild, uint32_t cost ) nodeToBuild->m_Stamp = 0; // Regenerate dynamic dependencies - if ( nodeToBuild->DoDynamicDependencies( *this, forceClean ) == false ) + if ( nodeToBuild->DoDynamicDependencies( *this ) == false ) { nodeToBuild->SetState( Node::FAILED ); return; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index 4e5a381e0..8b83ad2ba 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -344,10 +344,8 @@ ObjectListNode::~ObjectListNode() = default; // GatherDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool ObjectListNode::GatherDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) +/*virtual*/ bool ObjectListNode::GatherDynamicDependencies( NodeGraph & nodeGraph ) { - (void)forceClean; // dynamic deps are always re-added here, so this is meaningless - // clear dynamic deps from previous passes m_DynamicDependencies.Clear(); @@ -513,9 +511,9 @@ ObjectListNode::~ObjectListNode() = default; // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool ObjectListNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) +/*virtual*/ bool ObjectListNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { - if ( GatherDynamicDependencies( nodeGraph, forceClean ) == false ) + if ( GatherDynamicDependencies( nodeGraph ) == false ) { return false; // GatherDynamicDependencies will have emitted error } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h index afce005ea..feda55d52 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h @@ -49,8 +49,8 @@ class ObjectListNode : public Node protected: friend class FunctionObjectList; - virtual bool GatherDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ); - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool GatherDynamicDependencies( NodeGraph & nodeGraph ); + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; // internal helpers diff --git a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp index 022fc1218..947c9b90d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp @@ -126,7 +126,7 @@ const char * TestNode::GetEnvironmentString() const // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool TestNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool /*forceClean*/ ) +/*virtual*/ bool TestNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { // clear dynamic deps from previous passes m_DynamicDependencies.Clear(); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.h b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.h index fd497adca..932bde804 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.h @@ -26,7 +26,7 @@ class TestNode : public FileNode const char * GetEnvironmentString() const; private: - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; void EmitCompilationMessage( const char * workingDir ) const; From 49fcf5206ef72afd3257483530a3d7dba609e1df Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:28:00 +0930 Subject: [PATCH 049/129] TestExe: Remove redundant test - TestNodeGraph tests basic creation of this node already --- .../Tools/FBuild/FBuildTest/Tests/TestExe.cpp | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp index de6136319..837312b1e 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp @@ -19,7 +19,6 @@ class TestExe : public FBuildTest private: DECLARE_TESTS - void CreateNode() const; void Build() const; void CheckValidExe() const; void Build_NoRebuild() const; @@ -28,30 +27,11 @@ class TestExe : public FBuildTest // Register Tests //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestExe ) - REGISTER_TEST( CreateNode ) REGISTER_TEST( Build ) REGISTER_TEST( CheckValidExe ) REGISTER_TEST( Build_NoRebuild ) REGISTER_TESTS_END -// CreateNode -//------------------------------------------------------------------------------ -void TestExe::CreateNode() const -{ - FBuild fb; - NodeGraph ng; - - #if defined( __WINDOWS__ ) - AStackString<> exeName( "c:\\exe.exe" ); - #else - AStackString<> exeName( "/tmp/exe.exe" ); - #endif - const ExeNode * exeNode = ng.CreateNode( exeName ); - TEST_ASSERT( exeNode->GetType() == Node::EXE_NODE ); - TEST_ASSERT( ExeNode::GetTypeS() == Node::EXE_NODE ); - TEST_ASSERT( AStackString<>( "Exe" ) == exeNode->GetTypeName() ); -} - // Build //------------------------------------------------------------------------------ void TestExe::Build() const From f7497943c1ac41f18ab8472888e5250ad304cb1e Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:30:30 +0930 Subject: [PATCH 050/129] [Improvement] Error #1100 emits location of previous declaration - Track the token associated with the creation of targets where possible - When emitting error 1100 emit additional information about the previous declaration if possible --- .../FBuildCore/BFF/Functions/Function.cpp | 42 +++++++------ .../FBuildCore/BFF/Functions/Function.h | 5 +- .../FBuildCore/BFF/Functions/FunctionCopy.cpp | 11 ++-- .../BFF/Functions/FunctionRemoveDir.cpp | 7 ++- .../BFF/Functions/FunctionSettings.cpp | 7 ++- Code/Tools/FBuild/FBuildCore/Error.cpp | 7 ++- Code/Tools/FBuild/FBuildCore/Error.h | 3 +- .../FBuild/FBuildCore/Graph/LinkerNode.cpp | 4 +- .../FBuild/FBuildCore/Graph/NodeGraph.cpp | 60 +++++++++++++++++-- .../Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 17 ++++-- .../FBuildCore/Graph/ObjectListNode.cpp | 2 +- .../FBuildTest/Tests/TestBFFParsing.cpp | 31 ++++++++++ 12 files changed, 153 insertions(+), 43 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp index ebeca26e9..3aac50c63 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp @@ -210,6 +210,7 @@ Function::~Function() = default; { return false; // substitution will have emitted an error } + m_AliasForFunctionSourceToken = headerArgsIter; ++headerArgsIter; } else if ( headerArgsIter->IsVariable() ) @@ -247,6 +248,7 @@ Function::~Function() = default; // Store alias name for use in Commit m_AliasForFunction = varSrc->GetString(); + m_AliasForFunctionSourceToken = headerArgsIter; ++headerArgsIter; } @@ -324,11 +326,14 @@ Function::~Function() = default; const bool aliasUsedForName = nameFromMetaData.IsEmpty(); AString & name = ( aliasUsedForName ) ? m_AliasForFunction : nameFromMetaData; ASSERT( name.IsEmpty() == false ); + const BFFToken * nameSourceToken = aliasUsedForName ? m_AliasForFunctionSourceToken + : funcStartIter; // TODO:C This could probably be improved // Check name isn't already used - if ( nodeGraph.FindNode( name ) ) + if ( const Node * existingNode = nodeGraph.FindNode( name ) ) { - Error::Error_1100_AlreadyDefined( funcStartIter, this, name ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( nameSourceToken, this, name, existingToken ); FDELETE node; return false; } @@ -337,7 +342,7 @@ Function::~Function() = default; node->SetName( Move( name ) ); // Register with NodeGraph - nodeGraph.RegisterNode( node ); + nodeGraph.RegisterNode( node, nameSourceToken ); // Set properties if ( !PopulateProperties( nodeGraph, funcStartIter, node ) ) @@ -358,7 +363,7 @@ Function::~Function() = default; } // handle alias creation - return ProcessAlias( nodeGraph, funcStartIter, node ); + return ProcessAlias( nodeGraph, node ); } // GetString @@ -562,7 +567,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, Node * node = nodeGraph.FindNode( name ); if ( node == nullptr ) { - DirectoryListNode * dln = nodeGraph.CreateNode( name ); + DirectoryListNode * dln = nodeGraph.CreateNode( name, iter ); node = dln; dln->m_Path = path; if ( patterns ) @@ -645,7 +650,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, } // Implicitly create the new node - compilerNode = nodeGraph.CreateNode( nodeName ); + compilerNode = nodeGraph.CreateNode( nodeName, iter ); VERIFY( compilerNode->GetReflectionInfoV()->SetProperty( compilerNode, "Executable", compiler ) ); VERIFY( compilerNode->GetReflectionInfoV()->SetProperty( compilerNode, "AllowDistribution", false ) ); const char * lastSlash = compiler.FindLast( NATIVE_SLASH ); @@ -676,7 +681,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, Node * node = nodeGraph.FindNode( file ); if ( node == nullptr ) { - node = nodeGraph.CreateNode( file ); + node = nodeGraph.CreateNode( file, iter ); } else if ( node->IsAFile() == false ) { @@ -786,7 +791,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, if ( n == nullptr ) { // not found - create a new file node - n = nodeGraph.CreateNode( nodeName ); + n = nodeGraph.CreateNode( nodeName, iter ); nodes.Add( n ); return true; } @@ -905,16 +910,16 @@ bool Function::GetStrings( const BFFToken * iter, Array< AString > & strings, co // ProcessAlias //------------------------------------------------------------------------------ -bool Function::ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Node * nodeToAlias ) const +bool Function::ProcessAlias( NodeGraph & nodeGraph, Node * nodeToAlias ) const { Dependencies nodesToAlias( 1 ); nodesToAlias.Add( nodeToAlias ); - return ProcessAlias( nodeGraph, iter, nodesToAlias ); + return ProcessAlias( nodeGraph, nodesToAlias ); } // ProcessAlias //------------------------------------------------------------------------------ -bool Function::ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Dependencies & nodesToAlias ) const +bool Function::ProcessAlias( NodeGraph & nodeGraph, Dependencies & nodesToAlias ) const { if ( m_AliasForFunction.IsEmpty() ) { @@ -922,18 +927,20 @@ bool Function::ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Depen } // check for duplicates - if ( nodeGraph.FindNode( m_AliasForFunction ) ) + if ( const Node * existingNode = nodeGraph.FindNode( m_AliasForFunction ) ) { - Error::Error_1100_AlreadyDefined( iter, this, m_AliasForFunction ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( m_AliasForFunctionSourceToken, this, m_AliasForFunction, existingToken ); return false; } // create an alias against the node - AliasNode * an = nodeGraph.CreateNode( m_AliasForFunction ); + AliasNode * an = nodeGraph.CreateNode( m_AliasForFunction, m_AliasForFunctionSourceToken ); an->m_StaticDependencies = nodesToAlias; // TODO: make this use m_Targets & Initialize() - // clear the string so it can't be used again + // Clear the alias info so it can't be used again m_AliasForFunction.Clear(); + m_AliasForFunctionSourceToken = nullptr; return true; } @@ -982,9 +989,10 @@ bool Function::GetNameForNode( NodeGraph & nodeGraph, const BFFToken * iter, con } // Check that name isn't already used - if ( nodeGraph.FindNode( strings[0] ) ) + if ( const Node * existingNode = nodeGraph.FindNode( strings[0] ) ) { - Error::Error_1100_AlreadyDefined( iter, this, strings[0] ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( iter, this, strings[0], existingToken ); return false; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.h b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.h index 1244ceb76..24464e7b6 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.h @@ -126,6 +126,7 @@ class Function // for functions that support a simple alias parameter, the base class can // parse it out mutable AString m_AliasForFunction; + mutable const BFFToken * m_AliasForFunctionSourceToken = nullptr; // Helpers to get nodes bool GetNodeList( NodeGraph & nodeGraph, @@ -145,8 +146,8 @@ class Function bool GetStrings( const BFFToken * iter, Array< AString > & strings, const char * name, bool required = false ) const; // helper function to make alias for target - bool ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Node * nodeToAlias ) const; - bool ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Dependencies & nodesToAlias ) const; + bool ProcessAlias( NodeGraph & nodeGraph, Node * nodeToAlias ) const; + bool ProcessAlias( NodeGraph & nodeGraph, Dependencies & nodesToAlias ) const; // Reflection based property population bool GetNameForNode( NodeGraph & nodeGraph, const BFFToken *iter, const ReflectionInfo * ri, AString & name ) const; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp index 7feebc2f7..0cc86e29c 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp @@ -104,7 +104,7 @@ FunctionCopy::FunctionCopy() else { // source file not defined by use - assume an external file - srcNodes.Append( nodeGraph.CreateNode( *it ) ); + srcNodes.Append( nodeGraph.CreateNode( *it, funcStartIter ) ); } } } @@ -148,14 +148,15 @@ FunctionCopy::FunctionCopy() } // check node doesn't already exist - if ( nodeGraph.FindNode( dst ) ) + if ( const Node * existingNode = nodeGraph.FindNode( dst ) ) { - Error::Error_1100_AlreadyDefined( funcStartIter, this, dst ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( funcStartIter, this, dst, existingToken ); return false; } // create our node - CopyFileNode * copyFileNode = nodeGraph.CreateNode( dst ); + CopyFileNode * copyFileNode = nodeGraph.CreateNode( dst, funcStartIter ); copyFileNode->m_Source = srcNode->GetName(); copyFileNode->m_PreBuildDependencyNames = preBuildDependencyNames; if ( !copyFileNode->Initialize( nodeGraph, funcStartIter, this ) ) @@ -167,7 +168,7 @@ FunctionCopy::FunctionCopy() } // handle alias creation - return ProcessAlias( nodeGraph, funcStartIter, copyNodes ); + return ProcessAlias( nodeGraph, copyNodes ); } // GetSourceNodes diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp index 30fa7e982..076e81a19 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp @@ -36,13 +36,14 @@ FunctionRemoveDir::FunctionRemoveDir() //------------------------------------------------------------------------------ /*virtual*/ bool FunctionRemoveDir::Commit( NodeGraph & nodeGraph, const BFFToken * funcStartIter ) const { - if ( nodeGraph.FindNode( m_AliasForFunction ) ) + if ( const Node * existingNode = nodeGraph.FindNode( m_AliasForFunction ) ) { - Error::Error_1100_AlreadyDefined( funcStartIter, this, m_AliasForFunction ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( funcStartIter, this, m_AliasForFunction, existingToken ); return false; } - Node * removeDirNode = nodeGraph.CreateNode( m_AliasForFunction ); + Node * removeDirNode = nodeGraph.CreateNode( m_AliasForFunction, funcStartIter ); if ( !PopulateProperties( nodeGraph, funcStartIter, removeDirNode ) ) { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp index 5e7084262..b9309475b 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp @@ -28,13 +28,14 @@ FunctionSettings::FunctionSettings() /*virtual*/ bool FunctionSettings::Commit( NodeGraph & nodeGraph, const BFFToken * funcStartIter ) const { AStackString<> name( "$$Settings$$" ); - if ( nodeGraph.FindNode( name ) ) + if ( const Node * existingNode = nodeGraph.FindNode( name ) ) { - Error::Error_1100_AlreadyDefined( funcStartIter, this, name ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( funcStartIter, this, name, existingToken ); return false; } - SettingsNode * settingsNode = nodeGraph.CreateNode( name ); + SettingsNode * settingsNode = nodeGraph.CreateNode( name, funcStartIter ); if ( !PopulateProperties( nodeGraph, funcStartIter, settingsNode ) ) { diff --git a/Code/Tools/FBuild/FBuildCore/Error.cpp b/Code/Tools/FBuild/FBuildCore/Error.cpp index 0fceffdf3..4b2dc15c1 100644 --- a/Code/Tools/FBuild/FBuildCore/Error.cpp +++ b/Code/Tools/FBuild/FBuildCore/Error.cpp @@ -477,10 +477,15 @@ //------------------------------------------------------------------------------ /*static*/ void Error::Error_1100_AlreadyDefined( const BFFToken * iter, const Function * function, - const AString & name ) + const AString & name, + const BFFToken * previousDeclarationToken ) { FormatError( iter, 1100u, function, "Target '%s' already defined.", name.Get() ); + if ( previousDeclarationToken ) + { + FormatError( previousDeclarationToken, 1100u, nullptr, "Previously declared here:" ); + } } // Error_1101_MissingProperty diff --git a/Code/Tools/FBuild/FBuildCore/Error.h b/Code/Tools/FBuild/FBuildCore/Error.h index 2556aa0c1..fa38ddf2a 100644 --- a/Code/Tools/FBuild/FBuildCore/Error.h +++ b/Code/Tools/FBuild/FBuildCore/Error.h @@ -138,7 +138,8 @@ class Error //------------------------------------------------------------------------------ static void Error_1100_AlreadyDefined( const BFFToken * iter, const Function * function, - const AString & name ); + const AString & name, + const BFFToken * previousDeclarationToken = nullptr ); static void Error_1101_MissingProperty( const BFFToken * iter, const Function * function, const AString & name ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp index ec4e2f06d..656580a2f 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp @@ -1378,7 +1378,7 @@ void LinkerNode::GetImportLibName( const AString & args, AString & importLibName // see if the file exists on disk at this location if ( LinkerNodeFileExistsCache::Get().FileExists( potentialNodeNameClean ) ) { - node = nodeGraph.CreateNode( potentialNodeNameClean ); + node = nodeGraph.CreateNode( potentialNodeNameClean, iter ); libs.Add( node ); found = true; FLOG_VERBOSE( "Additional library '%s' assumed to be '%s'\n", lib.Get(), potentialNodeNameClean.Get() ); @@ -1519,7 +1519,7 @@ void LinkerNode::GetImportLibName( const AString & args, AString & importLibName // node not found - create a new FileNode, assuming we are // linking against an externally built library - node = nodeGraph.CreateNode( nodeName ); + node = nodeGraph.CreateNode( nodeName, iter ); nodes.Add( node ); return true; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index 9f97b673d..9d6cb5a29 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -213,7 +213,7 @@ bool NodeGraph::ParseFromRoot( const char * bffFile ) const AStackString<> settingsNodeName( "$$Settings$$" ); const Node * settingsNode = FindNode( settingsNodeName ); m_Settings = settingsNode ? settingsNode->CastTo() - : CreateNode( settingsNodeName ); // Create a default + : CreateNode( settingsNodeName, &BFFToken::GetBuiltInToken() ); // Create a default // Parser will populate m_UsedFiles const Array & usedFiles = bffParser.GetUsedFiles(); @@ -223,6 +223,11 @@ bool NodeGraph::ParseFromRoot( const char * bffFile ) m_UsedFiles.EmplaceBack( file->GetFileName(), file->GetTimeStamp(), file->GetHash() ); } } + + // Free token tracking data we no longer need (and won't be valid when + // BFFParser falls out of scope) + m_NodeSourceTokens.Destruct(); + return ok; } @@ -786,12 +791,38 @@ size_t NodeGraph::GetNodeCount() const // RegisterNode //------------------------------------------------------------------------------ -void NodeGraph::RegisterNode( Node * node ) +void NodeGraph::RegisterNode( Node * node, const BFFToken * sourceToken ) { ASSERT( Thread::IsMainThread() ); ASSERT( node->GetName().IsEmpty() == false ); ASSERT( FindNode( node->GetName() ) == nullptr ); AddNode( node ); + RegisterSourceToken( node, sourceToken ); +} + +// RegisterSourceToken +//------------------------------------------------------------------------------ +void NodeGraph::RegisterSourceToken( const Node * node, const BFFToken * sourceToken ) +{ + // Where available, record the source token for the node + if ( sourceToken ) + { + // Ammortize array growth in parallel with m_AllNodes + m_NodeSourceTokens.SetCapacity( m_AllNodes.GetCapacity() ); + + // Nobody should have added this before + ASSERT( m_NodeSourceTokens.GetSize() < m_AllNodes.GetSize() ); + + // Array may be non-contiguous, so fill it in + while ( m_NodeSourceTokens.GetSize() < m_AllNodes.GetSize() ) + { + m_NodeSourceTokens.EmplaceBack( nullptr ); + } + + // Store the token in the parallel at the same place as the node + ASSERT( m_AllNodes.Top() == node ); (void)node; + m_NodeSourceTokens.Top() = sourceToken; + } } // CreateNode @@ -850,7 +881,7 @@ Node * NodeGraph::CreateNode( Node::Type type, AString && name ) // CreateNode //------------------------------------------------------------------------------ -Node * NodeGraph::CreateNode( Node::Type type, const AString & name ) +Node * NodeGraph::CreateNode( Node::Type type, const AString & name, const BFFToken * sourceToken ) { // Where possible callers should call the move version to transfer ownership // of strings, but calers don't always have a string to transfer so this @@ -872,7 +903,11 @@ Node * NodeGraph::CreateNode( Node::Type type, const AString & name ) nameCopy = name; } - return CreateNode( type, Move( nameCopy ) ); + Node * node = CreateNode( type, Move( nameCopy ) ); + + RegisterSourceToken( node, sourceToken ); + + return node; } // AddNode @@ -1151,6 +1186,23 @@ void NodeGraph::SetBuildPassTagForAllNodes( uint32_t value ) const } } +// FindNodeSourceToken +//------------------------------------------------------------------------------ +const BFFToken * NodeGraph::FindNodeSourceToken( const Node * node ) const +{ + // Find the index of the node. This is a slow operation, but we only expect + // this to happen in non-critical paths like when emitting BFF parsing errors + const size_t index = m_AllNodes.GetIndexOf( m_AllNodes.Find( node ) ); + + // Return token if available. Not all nodes have creation info available. + if ( index < m_NodeSourceTokens.GetSize() ) + { + return m_NodeSourceTokens[ index ]; + } + + return nullptr; +} + // CleanPath //------------------------------------------------------------------------------ /*static*/ void NodeGraph::CleanPath( AString & name, bool makeFullPath ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index 36ed1ae24..14eadd3ee 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -110,15 +110,18 @@ class NodeGraph size_t GetNodeCount() const; const SettingsNode * GetSettings() const { return m_Settings; } - void RegisterNode( Node * n ); + void RegisterNode( Node * n, const BFFToken * sourceToken ); // create new nodes Node * CreateNode( Node::Type type, AString && name ); - Node * CreateNode( Node::Type type, const AString & name ); + Node * CreateNode( Node::Type type, + const AString & name, + const BFFToken * sourceToken = nullptr ); template - T * CreateNode( const AString & name ) + T * CreateNode( const AString & name, + const BFFToken * sourceToken = nullptr ) { - return CreateNode( T::GetTypeS(), name )->template CastTo(); + return CreateNode( T::GetTypeS(), name, sourceToken )->template CastTo(); } void DoBuildPass( Node * nodeToBuild ); @@ -126,6 +129,8 @@ class NodeGraph // Non-build operations that use the BuildPassTag can set it to a known value void SetBuildPassTagForAllNodes( uint32_t value ) const; + const BFFToken * FindNodeSourceToken( const Node * node ) const; + static void CleanPath( AString & name, bool makeFullPath = true ); static void CleanPath( const AString & name, AString & cleanPath, bool makeFullPath = true ); #if defined( ASSERTS_ENABLED ) @@ -175,6 +180,8 @@ class NodeGraph bool & movedDB ) const; uint32_t GetLibEnvVarHash() const; + void RegisterSourceToken( const Node * node, const BFFToken * sourceToken ); + // load/save helpers static void SerializeToText( Node * node, uint32_t depth, AString & outBuffer ); static void SerializeToText( const char * title, const Dependencies & dependencies, uint32_t depth, AString & outBuffer ); @@ -216,6 +223,8 @@ class NodeGraph }; Array< UsedFile > m_UsedFiles; + Array< const BFFToken * > m_NodeSourceTokens; + const SettingsNode * m_Settings; static uint32_t s_BuildPassTag; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index 8b83ad2ba..9c089bb8f 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -775,7 +775,7 @@ ObjectNode * ObjectListNode::CreateObjectNode( NodeGraph & nodeGraph, const AString & objectInput, const AString & pchObjectName ) { - ObjectNode * node= nodeGraph.CreateNode( objectName ); + ObjectNode * node= nodeGraph.CreateNode( objectName, iter ); node->m_Compiler = m_Compiler; node->m_CompilerOptions = compilerOptions; node->m_CompilerOptionsDeoptimized = compilerOptionsDeoptimized; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp index 2d79d76e0..16004de70 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp @@ -81,6 +81,7 @@ class TestBFFParsing : public FBuildTest void ErrorRowAndColumn() const; void ForEach() const; void FunctionHeaders() const; + void AlreadyDefined() const; }; // Register Tests @@ -145,6 +146,7 @@ REGISTER_TESTS_BEGIN( TestBFFParsing ) REGISTER_TEST( ErrorRowAndColumn ) REGISTER_TEST( ForEach ) REGISTER_TEST( FunctionHeaders ) + REGISTER_TEST( AlreadyDefined ) REGISTER_TESTS_END // Empty @@ -1097,3 +1099,32 @@ void TestBFFParsing::FunctionHeaders() const } //------------------------------------------------------------------------------ +void TestBFFParsing::AlreadyDefined() const +{ + // Alias + TEST_PARSE_FAIL( "Alias( 'X' ) { .Targets = 'file' }\n" + "Alias( 'X' ) { .Targets = 'file' }\n", + "Error #1100 - Previously declared here:" ); + + // Alias for a Function + TEST_PARSE_FAIL( "Test( 'X' ) { .TestExecutable = 'exe' .TestOutput = 'out1' }\n" + "Test( 'X' ) { .TestExecutable = 'exe' .TestOutput = 'out2' }\n", + "Error #1100 - Previously declared here:" ); + + // Direct name (not an Alias) + TEST_PARSE_FAIL( "ObjectList( 'X' ) { .Compiler = 'clang' .CompilerOptions = '%1 %2' }\n" + "ObjectList( 'X' ) { .Compiler = 'clang' .CompilerOptions = '%1 %2' }\n", + "Error #1100 - Previously declared here:" ); + + // Aliasing output + TEST_PARSE_FAIL( "Alias( 'X' ) { .Targets = 'file' }\n" + "TextFile() { .TextFileOutput = 'file' }\n", + "Error #1100 - Previously declared here:" ); + + // Same target created multiple times in a loop + TEST_PARSE_FAIL( ".Y = { 'a', 'b' }\n" + "ForEach( .X in .Y ) { Test() { .TestExecutable = 'exe' .TestOutput = 'out' } }\n", + "Error #1100 - Previously declared here:" ); +} + +//------------------------------------------------------------------------------ From 0c6d1046e12b031e0bdd71ed8a7ba56efc7429aa Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Wed, 5 Jul 2023 14:58:37 +0930 Subject: [PATCH 051/129] [Improvement] All directory listings optimized slightly - walk each directory only once (instead of once for files and once for folders) - move strings into final destination instead of copying [Improvement] Directory listings with excluded folders optimized significantly - filter directories as the tree is walked instead of after, eliminating recursion into sub-directories --- Code/Core/FileIO/FileIO.cpp | 195 ++++++++++++++++++ Code/Core/FileIO/FileIO.h | 33 +++ .../FBuildCore/Graph/DirectoryListNode.cpp | 141 ++++++++----- .../FBuildCore/Graph/DirectoryListNode.h | 2 +- 4 files changed, 313 insertions(+), 58 deletions(-) diff --git a/Code/Core/FileIO/FileIO.cpp b/Code/Core/FileIO/FileIO.cpp index 3981ca337..97ee6fab8 100644 --- a/Code/Core/FileIO/FileIO.cpp +++ b/Code/Core/FileIO/FileIO.cpp @@ -321,6 +321,17 @@ return ( results->GetSize() != oldSize ); } +// GetFiles +//------------------------------------------------------------------------------ +/*static*/ void FileIO::GetFiles( const AString & path, + GetFilesHelper & helper ) +{ + // make a copy of the path as it will be modified during recursion + AStackString<> pathCopy( path ); + PathUtils::EnsureTrailingSlash( pathCopy ); + GetFilesRecurse( pathCopy, helper ); +} + // GetFilesEx //------------------------------------------------------------------------------ /*static*/ bool FileIO::GetFilesEx( const AString & path, @@ -964,6 +975,150 @@ } #endif +//------------------------------------------------------------------------------ +/*static*/ void FileIO::GetFilesRecurse( AString & pathCopy, GetFilesHelper & helper ) +{ + const uint32_t baseLength = pathCopy.GetLength(); + + // Windows requires the path contain a wildcard + #if defined( __WINDOWS__ ) + pathCopy += '*'; // retrieve all files/folders + #endif + + // Don't traverse into symlinks - TODO:B Why is this OSX/Linux only? + #if defined( __LINUX__ ) || defined( __APPLE__ ) + // Special case symlinks + struct stat stat_source; + if ( lstat( pathCopy.Get(), &stat_source ) != 0 ) + { + return; + } + if ( S_ISLNK( stat_source.st_mode ) ) + { + return; + } + #endif + + // Start dir list operation + #if defined( __WINDOWS__ ) + WIN32_FIND_DATA findData; + HANDLE hFind = FindFirstFileEx( pathCopy.Get(), FindExInfoBasic, &findData, FindExSearchNameMatch, nullptr, 0 ); + if ( hFind == INVALID_HANDLE_VALUE) + { + return; + } + #else + DIR * dir = opendir( pathCopy.Get() ); + if ( dir == nullptr ) + { + return; + } + #endif + + // Iterate entries + #if defined( __LINUX__ ) || defined( __APPLE__ ) + for ( ;; ) + #else + do + #endif + { + #if defined( __LINUX__ ) || defined( __APPLE__ ) + dirent * entry = readdir( dir ); + if ( entry == nullptr ) + { + break; // no more entries + } + #endif + + // Name of this entry + #if defined( __WINDOWS__ ) + const char * const entryName = findData.cFileName; + #else + const char * const entryName = entry->d_name; + #endif + + // Determine if entry is a directory + #if defined( __WINDOWS__ ) + const bool isDir = ( ( findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) == FILE_ATTRIBUTE_DIRECTORY ); + #else + bool isDir = ( entry->d_type == DT_DIR ); + + // Not all filesystems have support for returning the file type in + // d_type and applications must properly handle a return of DT_UNKNOWN. + if ( entry->d_type == DT_UNKNOWN ) + { + pathCopy.SetLength( baseLength ); + pathCopy += entry->d_name; + + struct stat info; + VERIFY( lstat( pathCopy.Get(), &info ) == 0 ); + isDir = S_ISDIR( info.st_mode ); + } + #endif + + // Directory? + if ( isDir ) + { + // ignore magic '.' and '..' folders + // (don't need to check length of name, as all names are at least 1 char + // which means index 0 and 1 are valid to access) + if ( ( entryName[ 0 ] == '.' ) && + ( ( entryName[ 1 ] == '.' ) || ( entryName[ 1 ] == 0 ) ) ) + { + continue; + } + + // Process Dir + pathCopy.SetLength( baseLength ); + pathCopy += entryName; + pathCopy += NATIVE_SLASH; + const bool recurseIntoDir = helper.OnDirectory( pathCopy ); + if ( recurseIntoDir ) + { + GetFilesRecurse( pathCopy, helper ); + } + continue; + } + + // Process File + pathCopy.SetLength( baseLength ); + if ( helper.ShouldIncludeFile( entryName ) ) + { + FileInfo fileInfo; + const uint32_t fileNameLen = static_cast( AString::StrLen( entryName ) ); + const uint32_t fullPathLen = pathCopy.GetLength() + fileNameLen; + fileInfo.m_Name.SetReserved( fullPathLen ); + fileInfo.m_Name.Assign( pathCopy ); + fileInfo.m_Name.Append(entryName, fileNameLen ); + #if defined( __WINDOWS__ ) + fileInfo.m_Attributes = findData.dwFileAttributes; + fileInfo.m_LastWriteTime = (uint64_t)findData.ftLastWriteTime.dwLowDateTime | ( (uint64_t)findData.ftLastWriteTime.dwHighDateTime << 32 ); + fileInfo.m_Size = (uint64_t)findData.nFileSizeLow | ( (uint64_t)findData.nFileSizeHigh << 32 ); + #else + struct stat info; + VERIFY( lstat( fileInfo.m_Name.Get(), &info ) == 0 ); + fileInfo.m_Attributes = info.st_mode; + #if defined( __APPLE__ ) + fileInfo.m_LastWriteTime = ( ( (uint64_t)info.st_mtimespec.tv_sec * 1000000000ULL ) + (uint64_t)info.st_mtimespec.tv_nsec ); + #else + fileInfo.m_LastWriteTime = ( ( (uint64_t)info.st_mtim.tv_sec * 1000000000ULL ) + (uint64_t)info.st_mtim.tv_nsec ); + #endif + fileInfo.m_Size = info.st_size; + #endif + helper.OnFile( Move( fileInfo ) ); + } + } + #if defined( __WINDOWS__ ) + while ( FindNextFile( hFind, &findData ) != 0 ); + #endif + + #if defined( __WINDOWS__ ) + FindClose( hFind ); + #else + closedir( dir ); + #endif +} + // GetFilesRecurse //------------------------------------------------------------------------------ /*static*/ void FileIO::GetFilesRecurse( AString & pathCopy, @@ -1608,4 +1763,44 @@ bool FileIO::FileInfo::IsReadOnly() const return false; } +// GetFilesHelper CONSTRUCTOR +//------------------------------------------------------------------------------ +GetFilesHelper::GetFilesHelper( size_t sizeHint ) + : m_Files( sizeHint, true ) +{ +} + +//------------------------------------------------------------------------------ +GetFilesHelper::GetFilesHelper( const Array & patterns, size_t sizeHint ) + : m_Patterns( &patterns ) + , m_Files( sizeHint, true ) +{ +} + +// GetFilesHelper DESTRUCTOR +//------------------------------------------------------------------------------ +GetFilesHelper::~GetFilesHelper() = default; + +// OnDirectory +//------------------------------------------------------------------------------ +/*virtual*/ bool GetFilesHelper::OnDirectory( const AString & /*dirPath*/ ) +{ + return m_Recurse; +} + +// ShouldIncludeFile +//------------------------------------------------------------------------------ +/*virtual*/ bool GetFilesHelper::ShouldIncludeFile( const char * fileName ) +{ + // Check wildcard patterns + return FileIO::IsMatch( m_Patterns, fileName ); +} + +// OnFile +//------------------------------------------------------------------------------ +/*virtual*/ void GetFilesHelper::OnFile( FileIO::FileInfo && fileInfo ) +{ + m_Files.EmplaceBack( Move( fileInfo ) ); +} + //------------------------------------------------------------------------------ diff --git a/Code/Core/FileIO/FileIO.h b/Code/Core/FileIO/FileIO.h index 276de7d23..0e33a44df 100644 --- a/Code/Core/FileIO/FileIO.h +++ b/Code/Core/FileIO/FileIO.h @@ -12,6 +12,10 @@ //------------------------------------------------------------------------------ #define MAX_PATH 260 +// Forward Declarations +//------------------------------------------------------------------------------ +class GetFilesHelper; + // FileIO //------------------------------------------------------------------------------ class FileIO @@ -28,6 +32,8 @@ class FileIO const AString & wildCard, bool recurse, Array< AString > * results ); + static void GetFiles( const AString & path, + GetFilesHelper & helper ); struct FileInfo { AString m_Name; @@ -87,6 +93,8 @@ class FileIO static bool IsWindowsLongPathSupportEnabledInternal(); #endif + static void GetFilesRecurse( AString & path, + GetFilesHelper & helper ); static void GetFilesRecurse( AString & path, const AString & wildCard, Array< AString > * results ); @@ -99,7 +107,32 @@ class FileIO static void GetFilesNoRecurseEx( const char * path, const Array< AString > * patterns, Array< FileInfo > * results ); + + friend class GetFilesHelper; static bool IsMatch( const Array< AString > * patterns, const char * fileName ); }; //------------------------------------------------------------------------------ +class GetFilesHelper +{ +public: + explicit GetFilesHelper( size_t sizeHint = 1024 ); + explicit GetFilesHelper( const Array & patterns, size_t sizeHint = 1024 ); + virtual ~GetFilesHelper(); + + // Default implementation can be extended or customized + [[nodiscard]] virtual bool OnDirectory( const AString & dirPath ); + [[nodiscard]] virtual bool ShouldIncludeFile( const char * fileName ); + virtual void OnFile( FileIO::FileInfo && fileInfo ); + + // Access results + const Array & GetFiles() const { return m_Files; } + Array & GetFiles() { return m_Files; } + +protected: + bool m_Recurse = true; + const Array * m_Patterns = nullptr; + Array m_Files; +}; + +//------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp index e97976a27..8b83a74c7 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp @@ -14,6 +14,7 @@ #include "Core/FileIO/MemoryStream.h" #include "Core/FileIO/PathUtils.h" #include "Core/Math/xxHash.h" +#include "Core/Profile/Profile.h" #include "Core/Strings/AStackString.h" // Reflection @@ -28,6 +29,77 @@ REFLECT_NODE_BEGIN( DirectoryListNode, Node, MetaNone() ) REFLECT( m_IncludeReadOnlyStatusInHash, "IncludeReadOnlyStatusInHash", MetaHidden() ) REFLECT_END( DirectoryListNode ) +// DirectoryListNodeGetFilesHelper +//------------------------------------------------------------------------------ +class DirectoryListNodeGetFilesHelper : public GetFilesHelper +{ +public: + //-------------------------------------------------------------------------- + DirectoryListNodeGetFilesHelper( const Array & patterns, + const Array & excludePaths, + const Array & filesToExclude, + const Array & excludePatterns, + bool recurse ) + : GetFilesHelper( patterns ) + , m_ExcludePaths( excludePaths ) + , m_FilesToExclude( filesToExclude ) + , m_ExcludePatterns( excludePatterns ) + { + m_Recurse = recurse; + } + + //-------------------------------------------------------------------------- + virtual bool OnDirectory( const AString & path ) override + { + if ( m_Recurse == false ) + { + return false; + } + + // Filter excluded paths + for ( const AString & excludedPath : m_ExcludePaths ) + { + if ( PathUtils::PathBeginsWith( path, excludedPath ) ) + { + return false; // Don't recurse into dir + } + } + + return true; // Recurse into directory + } + + //-------------------------------------------------------------------------- + virtual void OnFile( FileIO::FileInfo && info ) override + { + // filter excluded files + for ( const AString & fileToExclude : m_FilesToExclude ) + { + if ( PathUtils::PathEndsWithFile( info.m_Name, fileToExclude ) ) + { + return; // Exclude + } + } + + // Filter excluded patterns + for ( const AString & excludePattern : m_ExcludePatterns ) + { + if ( PathUtils::IsWildcardMatch( excludePattern.Get(), info.m_Name.Get() ) ) + { + return; // Exclude + } + } + + // Keep file info + m_Files.EmplaceBack( Move( info ) ); + } + + DirectoryListNodeGetFilesHelper& operator =(DirectoryListNodeGetFilesHelper&) = delete; +protected: + const Array & m_ExcludePaths; + const Array & m_FilesToExclude; + const Array & m_ExcludePatterns; +}; + // CONSTRUCTOR //------------------------------------------------------------------------------ DirectoryListNode::DirectoryListNode() @@ -142,65 +214,20 @@ DirectoryListNode::~DirectoryListNode() = default; // NOTE: The DirectoryListNode makes no assumptions about whether no files // is an error or not. That's up to the dependent nodes to decide. - Array< FileIO::FileInfo > files( 4096, true ); - FileIO::GetFilesEx( m_Path, &m_Patterns, m_Recursive, &files ); - - m_Files.SetCapacity( files.GetSize() ); - - // filter exclusions - const FileIO::FileInfo * const end = files.End(); - for ( const FileIO::FileInfo * it = files.Begin(); it != end; it++ ) { - bool excluded = false; + // Get the list of files, filtered in various ways + DirectoryListNodeGetFilesHelper helper( m_Patterns, + m_ExcludePaths, + m_FilesToExclude, + m_ExcludePatterns, + m_Recursive ); + FileIO::GetFiles( m_Path, helper ); - // filter excluded paths - const AString * const eEnd = m_ExcludePaths.End(); - for ( const AString * eIt=m_ExcludePaths.Begin(); eIt != eEnd; ++eIt ) - { - if ( PathUtils::PathBeginsWith( it->m_Name, *eIt ) ) - { - excluded = true; - break; - } - } - - // filter excluded files - if ( !excluded ) - { - const AString * fit = m_FilesToExclude.Begin(); - const AString * const fend = m_FilesToExclude.End(); - for ( ; fit != fend; ++fit ) - { - if ( PathUtils::PathEndsWithFile( it->m_Name, *fit ) ) - { - excluded = true; - break; - } - } - } - - // filter excluded patterns - if ( !excluded ) - { - const AString * pit = m_ExcludePatterns.Begin(); - const AString * const pend = m_ExcludePatterns.End(); - for ( ; pit != pend; ++pit ) - { - if ( PathUtils::IsWildcardMatch( pit->Get(), it->m_Name.Get() ) ) - { - excluded = true; - break; - } - } - } - - if ( !excluded ) - { - m_Files.Append( *it ); - } + // Transfer ownership of filtered list + m_Files = Move( helper.GetFiles() ); } - MakePrettyName( files.GetSize() ); + MakePrettyName(); if ( FLog::ShowVerbose() ) { @@ -243,7 +270,7 @@ DirectoryListNode::~DirectoryListNode() = default; // MakePrettyName //------------------------------------------------------------------------------ -void DirectoryListNode::MakePrettyName( const size_t totalFiles ) +void DirectoryListNode::MakePrettyName() { AStackString<> prettyName( m_Path ); if (m_Recursive) @@ -252,7 +279,7 @@ void DirectoryListNode::MakePrettyName( const size_t totalFiles ) } const size_t numFiles = m_Files.GetSize(); - prettyName.AppendFormat( ", files kept: %zu / %zu", numFiles, totalFiles ); + prettyName.AppendFormat( ", files: %zu ", numFiles ); m_PrettyName = prettyName; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h index 0cb0cdf3e..30899b860 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h @@ -40,7 +40,7 @@ class DirectoryListNode : public Node private: virtual BuildResult DoBuild( Job * job ) override; - void MakePrettyName( const size_t totalFiles ); + void MakePrettyName(); friend class CompilationDatabase; // For DoBuild - TODO:C This is not ideal From e3507b0ebab3eac05d6017f91401325eb2f4d50a Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Fri, 7 Jul 2023 12:59:40 +0930 Subject: [PATCH 052/129] [Improvement] Improve LightCache performance considerably for many CPU core systems - Eliminate a large amount of redundant processing, particularly at the start of builds by extending lock duration to entire processing operation. The slight reduction in parallelism due to collisions is more than offset by the reduced redundant work. In testing, the host processing time was more than halved. - Increase number of buckets by 4x to reduce contention. This further offsets collision impact due to the above, but is also a gain without those changes. - Avoid some string copies by moving results into final destination - Ensure bucket structures are cacheline aligned (typically already the case, but this ensures layout is predictable) --- .../FBuild/FBuildCore/Cache/LightCache.cpp | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp index fd9fadad6..b4b4585ac 100644 --- a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp +++ b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp @@ -41,8 +41,8 @@ class IncludedFile class Include { public: - Include( const AString & include, IncludeType type ) - : m_Include( include ) + Include( AString && include, IncludeType type ) + : m_Include( Move( include ) ) , m_Type( type ) {} @@ -97,8 +97,7 @@ class IncludedFileHashSet return nullptr; } - // If two threads find the same include simultaneously, we delete the new - // one and return the old one. + // Insert a new item. Item must not already exist. const IncludedFile * Insert( IncludedFile * item ) { if ( ( m_Buckets.GetSize() / 2 ) <= m_Elts ) @@ -111,13 +110,7 @@ class IncludedFileHashSet IncludedFile ** location = InternalFind( item->m_FileName, item->m_FileNameHash ); ASSERT( location != nullptr ); - if ( *location != nullptr ) - { - // A race between multiple threads got us a duplicate item. - // delete the new one - FDELETE item; - return *location; - } + ASSERT( *location == nullptr ); // Item must not exist already ++m_Elts; *location = item; @@ -227,7 +220,8 @@ IncludedFile::~IncludedFile() // IncludedFileBucket //------------------------------------------------------------------------------ -class IncludedFileBucket +PRAGMA_DISABLE_PUSH_MSVC( 4324 ) // structure was padded due to alignment specifier +class alignas(64) IncludedFileBucket // Align to cache line boundary { public: IncludedFileBucket() = default; @@ -241,9 +235,9 @@ class IncludedFileBucket Mutex m_Mutex; IncludedFileHashSet m_HashSet; }; -// using a power of two number of buckets. 64 top level buckets should be a -// reasonable tradeoff between size and contention -#define LIGHTCACHE_NUM_BUCKET_BITS 6 +PRAGMA_DISABLE_POP_MSVC // 4324 +// Power of two number of buckets +#define LIGHTCACHE_NUM_BUCKET_BITS 9 // 512 buckets #define LIGHTCACHE_NUM_BUCKETS ( 1ULL << LIGHTCACHE_NUM_BUCKET_BITS ) #define LIGHTCACHE_BUCKET_MASK_BASE ( LIGHTCACHE_NUM_BUCKETS - 1ULL ) // use upper bits for bucket selection, as lower bits get used in the hash set @@ -446,7 +440,7 @@ bool LightCache::ParseDirective_Include( IncludedFile & file, const char * & pos SkipWhitespace( pos ); // Get include string - AStackString<> include; + AString include; if ( ( *pos == '"' ) || ( *pos == '<' ) ) { // Looks like a normal include @@ -458,12 +452,12 @@ bool LightCache::ParseDirective_Include( IncludedFile & file, const char * & pos return false; } - file.m_Includes.EmplaceBack( include, includeType ); + file.m_Includes.EmplaceBack( Move( include ), includeType ); return true; } // Not a normal include - perhaps this is a macro? - AStackString<> macroName; + AString macroName; if ( ParseMacroName( pos, macroName ) == false ) { // We saw an unexpected sequence after the #include @@ -472,7 +466,7 @@ bool LightCache::ParseDirective_Include( IncludedFile & file, const char * & pos } // Store the macro include which will be resolved later - file.m_Includes.EmplaceBack( macroName, IncludeType::MACRO ); + file.m_Includes.EmplaceBack( Move( macroName ), IncludeType::MACRO ); return true; } @@ -861,9 +855,23 @@ const IncludedFile * LightCache::FileExists( const AString & fileName ) const uint64_t fileNameHash = xxHash3::Calc64( fileName ); const uint64_t bucketIndex = LIGHTCACHE_HASH_TO_BUCKET( fileNameHash ); IncludedFileBucket & bucket = g_AllIncludedFiles[ bucketIndex ]; + + // Lock the bucket while we deal with this file. + // + // This prevents two threads processing the same file, which for most + // codebases would otherwise happen a lot (code tends to have a large set + // of shared headers). + // + // Collisions in the bucketIndex mean some unrelated files will block + // each other, but not redundantly processing the same file results + // in a significant speedup which more than offsets the blocking caused + // by collisions. LIGHTCACHE_NUM_BUCKET_BITS can be increased to further + // reduce collisions at the cost of memory. + // + MutexHolder mh( bucket.m_Mutex ); + // Retrieve from shared cache { - MutexHolder mh( bucket.m_Mutex ); const IncludedFile * location = bucket.m_HashSet.Find( fileName, fileNameHash ); if ( location ) { @@ -875,7 +883,6 @@ const IncludedFile * LightCache::FileExists( const AString & fileName ) // A newly seen file IncludedFile * newFile = FNEW( IncludedFile() ); - const IncludedFile * retval = nullptr; newFile->m_FileNameHash = fileNameHash; newFile->m_FileName = fileName; newFile->m_Exists = false; @@ -885,23 +892,16 @@ const IncludedFile * LightCache::FileExists( const AString & fileName ) FileStream f; if ( f.Open( fileName.Get() ) == false ) { - { - // Store to shared cache - MutexHolder mh( bucket.m_Mutex ); - retval = bucket.m_HashSet.Insert( newFile ); - } - return retval; + // Store to shared cache + return bucket.m_HashSet.Insert( newFile ); } // File exists - parse it newFile->m_Exists = true; Parse( newFile, f ); - { - // Store to shared cache - MutexHolder mh( bucket.m_Mutex ); - retval = bucket.m_HashSet.Insert( newFile ); - } + // Store to shared cache + const IncludedFile * retval = bucket.m_HashSet.Insert( newFile ); m_IncludeDefines.Append( retval->m_IncludeDefines ); From 699b2fc6c408b3b7da29958d7da072eb4fc7372e Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Fri, 7 Jul 2023 13:24:18 +0930 Subject: [PATCH 053/129] Job: Eliminate non-atomic access to mem usage during an assert --- Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp | 9 ++++----- Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp index 5ffa3661e..ea7d7dbf9 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp @@ -20,7 +20,7 @@ // Static //------------------------------------------------------------------------------ static uint32_t s_LastJobId( 0 ); -/*static*/ int64_t Job::s_TotalLocalDataMemoryUsage( 0 ); +/*static*/ Atomic Job::s_TotalLocalDataMemoryUsage( 0 ); // CONSTRUCTOR //------------------------------------------------------------------------------ @@ -88,8 +88,7 @@ void Job::OwnData( void * data, size_t size, bool compressed ) // Update total memory use tracking if ( m_IsLocal ) { - ASSERT( s_TotalLocalDataMemoryUsage >= m_DataSize ); - AtomicSub( &s_TotalLocalDataMemoryUsage, (int64_t)m_DataSize ); + VERIFY( s_TotalLocalDataMemoryUsage.Sub( static_cast( m_DataSize ) ) >= 0 ); } } @@ -101,7 +100,7 @@ void Job::OwnData( void * data, size_t size, bool compressed ) // Update total memory use tracking if ( m_IsLocal ) { - AtomicAdd( &s_TotalLocalDataMemoryUsage, (int64_t)m_DataSize ); + s_TotalLocalDataMemoryUsage.Add( static_cast( m_DataSize ) ); } } @@ -268,7 +267,7 @@ void Job::GetMessagesForMonitorLog( AString & buffer ) const //------------------------------------------------------------------------------ /*static*/ uint64_t Job::GetTotalLocalDataMemoryUsage() { - return (uint64_t)AtomicLoadRelaxed( &s_TotalLocalDataMemoryUsage ); + return static_cast( s_TotalLocalDataMemoryUsage.Load() ); } // SetBuildProfilerScope diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h index e55b81ce9..42e3f7442 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h @@ -6,6 +6,7 @@ //------------------------------------------------------------------------------ #include "Core/Env/MSVCStaticAnalysis.h" #include "Core/Env/Types.h" +#include "Core/Process/Atomic.h" #include "Core/Strings/AString.h" // Forward Declarations @@ -124,7 +125,7 @@ class Job Array< AString > m_Messages; - static int64_t s_TotalLocalDataMemoryUsage; // Total memory being managed by OwnData + static Atomic s_TotalLocalDataMemoryUsage; // Total memory being managed by OwnData }; //------------------------------------------------------------------------------ From 5570db0c0b16b56755a8dcb35c7deb7f733a4665 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 8 Jul 2023 10:39:43 +0930 Subject: [PATCH 054/129] NodeGraph::BuildRecurse: - convert multiple if statements into switch/case - eliminate states that were never used (always replaced) - rename states for clarity Should be functionally unchanged. --- Code/Tools/FBuild/FBuildCore/Graph/Node.h | 7 +- .../FBuild/FBuildCore/Graph/NodeGraph.cpp | 160 +++++++++--------- .../FBuild/FBuildCore/WorkerPool/JobQueue.cpp | 2 +- 3 files changed, 84 insertions(+), 85 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.h b/Code/Tools/FBuild/FBuildCore/Graph/Node.h index f7d788ca8..b98e250d6 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.h @@ -114,10 +114,9 @@ class Node : public Struct enum State : uint8_t { - NOT_PROCESSED, // no work done (either not part of this build, or waiting on static dependencies ) - PRE_DEPS_READY, // pre-build deps processed - STATIC_DEPS_READY, // static dependencies are uptodate - we are ready to DoDynamicDeps - DYNAMIC_DEPS_DONE, // dynamic deps updated, waiting for dynamic deps to be ready + NOT_PROCESSED, // no work done (either not part of this build, or not yet seen) + STATIC_DEPS, // pre-build deps processed and checking static deps + DYNAMIC_DEPS, // dynamic deps regenerated and being checked BUILDING, // in the queue for building FAILED, // failed to build UP_TO_DATE, // built, or confirmed as not needing building diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index 9d6cb5a29..822924ffc 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -993,108 +993,108 @@ void NodeGraph::BuildRecurse( Node * nodeToBuild, uint32_t cost ) { ASSERT( nodeToBuild ); - // already building, or queued to build? - ASSERT( nodeToBuild->GetState() != Node::BUILDING ); - // accumulate recursive cost cost += nodeToBuild->GetLastBuildTime(); - // check pre-build dependencies - if ( nodeToBuild->GetState() == Node::NOT_PROCESSED ) + // False positive "Unannotated fallthrough between switch labels" (VS 2019 v14.29.30037) + #if ( _MSC_VER < 1935 ) + PRAGMA_DISABLE_PUSH_MSVC(26819) + #endif + + switch ( nodeToBuild->GetState() ) { - // all pre-build deps done? - const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetPreBuildDependencies(), cost ); - if ( allDependenciesUpToDate == false ) + case Node::NOT_PROCESSED: { - return; // not ready or failed - } + // check pre-build dependencies + const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetPreBuildDependencies(), cost ); + if ( allDependenciesUpToDate == false ) + { + return; // not ready or failed + } + nodeToBuild->SetState( Node::STATIC_DEPS ); - nodeToBuild->SetState( Node::PRE_DEPS_READY ); - } + [[fallthrough]]; + } + case Node::STATIC_DEPS: + { + // check static dependencies + const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetStaticDependencies(), cost ); + if ( allDependenciesUpToDate == false ) + { + return; // not ready or failed + } - ASSERT( ( nodeToBuild->GetState() == Node::PRE_DEPS_READY ) || - ( nodeToBuild->GetState() == Node::STATIC_DEPS_READY ) || - ( nodeToBuild->GetState() == Node::DYNAMIC_DEPS_DONE ) ); + // If static deps require us to rebuild, dynamic dependencies need regenerating + if ( FBuild::Get().GetOptions().m_ForceCleanBuild || + nodeToBuild->DetermineNeedToBuildStatic() ) + { + // Explicitly mark node in a way that will result in it rebuilding should + // we cancel the build before builing this node + if ( nodeToBuild->m_Stamp == 0 ) + { + // Note that this is the first time we're building (since Node can't check + // stamp as we clear it below) + nodeToBuild->SetStatFlag( Node::STATS_FIRST_BUILD ); + } + nodeToBuild->m_Stamp = 0; - // test static dependencies first - if ( nodeToBuild->GetState() == Node::PRE_DEPS_READY ) - { - // all static deps done? - const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetStaticDependencies(), cost ); - if ( allDependenciesUpToDate == false ) - { - return; // not ready or failed - } + // Regenerate dynamic dependencies + nodeToBuild->m_DynamicDependencies.Clear(); + if ( nodeToBuild->DoDynamicDependencies( *this ) == false ) + { + nodeToBuild->SetState( Node::FAILED ); + return; + } - nodeToBuild->SetState( Node::STATIC_DEPS_READY ); - } + // Continue through to check dynamic dependencies and build + } - ASSERT( ( nodeToBuild->GetState() == Node::STATIC_DEPS_READY ) || - ( nodeToBuild->GetState() == Node::DYNAMIC_DEPS_DONE ) ); + // Dynamic dependencies are ready to be checked + nodeToBuild->SetState( Node::DYNAMIC_DEPS ); - if ( nodeToBuild->GetState() != Node::DYNAMIC_DEPS_DONE ) - { - // If static deps require us to rebuild, dynamic dependencies need regenerating - const bool forceClean = FBuild::Get().GetOptions().m_ForceCleanBuild; - if ( forceClean || - nodeToBuild->DetermineNeedToBuildStatic() ) + [[fallthrough]]; + } + case Node::DYNAMIC_DEPS: { - // Clear dynamic dependencies - nodeToBuild->m_DynamicDependencies.Clear(); - - // Explicitly mark node in a way that will result in it rebuilding should - // we cancel the build before builing this node - if ( nodeToBuild->m_Stamp == 0 ) + // check dynamic dependencies + const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetDynamicDependencies(), cost ); + if ( allDependenciesUpToDate == false ) { - // Note that this is the first time we're building (since Node can't check - // stamp as we clear it below) - nodeToBuild->SetStatFlag( Node::STATS_FIRST_BUILD ); + return; // not ready or failed } - nodeToBuild->m_Stamp = 0; - // Regenerate dynamic dependencies - if ( nodeToBuild->DoDynamicDependencies( *this ) == false ) + // dependencies are uptodate, so node can now tell us if it needs + // building + nodeToBuild->SetStatFlag( Node::STATS_PROCESSED ); + if ( ( nodeToBuild->GetStamp() == 0 ) || // Avoid redundant work in DetermineNeedToBuild + nodeToBuild->DetermineNeedToBuildDynamic() ) { - nodeToBuild->SetState( Node::FAILED ); - return; + nodeToBuild->m_RecursiveCost = cost; + JobQueue::Get().AddJobToBatch( nodeToBuild ); } - - // Continue through to check dynamic dependencies and build + else + { + if ( FLog::ShowVerbose() ) + { + FLOG_BUILD_REASON( "Up-To-Date '%s'\n", nodeToBuild->GetName().Get() ); + } + nodeToBuild->SetState( Node::UP_TO_DATE ); + } + break; } - - // Dynamic dependencies are ready to be checked - nodeToBuild->SetState( Node::DYNAMIC_DEPS_DONE ); - } - - ASSERT( nodeToBuild->GetState() == Node::DYNAMIC_DEPS_DONE ); - - // dynamic deps - { - // all static deps done? - const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetDynamicDependencies(), cost ); - if ( allDependenciesUpToDate == false ) + case Node::BUILDING: + case Node::FAILED: + case Node::UP_TO_DATE: { - return; // not ready or failed + ASSERT(false); // Should be impossible + break; } } - // dependencies are uptodate, so node can now tell us if it needs - // building - nodeToBuild->SetStatFlag( Node::STATS_PROCESSED ); - if ( ( nodeToBuild->GetStamp() == 0 ) || // Avoid redundant messages from DetermineNeedToBuild - nodeToBuild->DetermineNeedToBuildDynamic() ) - { - nodeToBuild->m_RecursiveCost = cost; - JobQueue::Get().AddJobToBatch( nodeToBuild ); - } - else - { - if ( FLog::ShowVerbose() ) - { - FLOG_BUILD_REASON( "Up-To-Date '%s'\n", nodeToBuild->GetName().Get() ); - } - nodeToBuild->SetState( Node::UP_TO_DATE ); - } + // False positive "Unannotated fallthrough between switch labels" (VS 2019 v14.29.30037) + #if ( _MSC_VER < 1935 ) + PRAGMA_DISABLE_POP_MSVC // 26819 + #endif } // CheckDependencies diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp index d29668c73..381c200a5 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp @@ -279,7 +279,7 @@ bool JobQueue::HasPendingCompletedJobs() const //------------------------------------------------------------------------------ void JobQueue::AddJobToBatch( Node * node ) { - ASSERT( node->GetState() == Node::DYNAMIC_DEPS_DONE ); + ASSERT( node->GetState() == Node::DYNAMIC_DEPS ); // mark as building node->SetState( Node::BUILDING ); From c1d994e8f6729f3415fae045d41a5374133475a5 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Jul 2023 13:10:30 +0930 Subject: [PATCH 055/129] Remove Clang 7 configs for FASTBuild on Windows: - we don't need to build FASTBuild itself on Windows with such an old version - the Clang versions FASTBuild can use is unchanged --- External/LZ4/LZ4.bff | 2 +- External/SDK/Clang/Clang.bff | 4 - External/SDK/Clang/Windows/Clang7.bff | 160 -------------------------- 3 files changed, 1 insertion(+), 165 deletions(-) delete mode 100644 External/SDK/Clang/Windows/Clang7.bff diff --git a/External/LZ4/LZ4.bff b/External/LZ4/LZ4.bff index 9f9ec1558..36cb9dc70 100644 --- a/External/LZ4/LZ4.bff +++ b/External/LZ4/LZ4.bff @@ -23,7 +23,7 @@ // When mixing old versions of Clang with VS2019, there are some problems with certian // bitscan intrinsics, so we disable their use here - #if USING_CLANG_7 || USING_CLANG_8 || USING_CLANG_9 + #if USING_CLANG_8 || USING_CLANG_9 .LZ4CompilerOptions + ' -DLZ4_FORCE_SW_BITCOUNT' #endif ] diff --git a/External/SDK/Clang/Clang.bff b/External/SDK/Clang/Clang.bff index 3cc75c65d..37012d415 100644 --- a/External/SDK/Clang/Clang.bff +++ b/External/SDK/Clang/Clang.bff @@ -13,7 +13,6 @@ #define USING_CLANG_12 #endif #if __WINDOWS__ && !CI_BUILD - //#define USING_CLANG_7 //#define USING_CLANG_8 //#define USING_CLANG_9 //#define USING_CLANG_10 @@ -53,9 +52,6 @@ #if CI_BUILD #define USING_CLANG_11 #endif - #if USING_CLANG_7 - #include "Windows/Clang7.bff" - #endif #if USING_CLANG_8 #include "Windows/Clang8.bff" #endif diff --git a/External/SDK/Clang/Windows/Clang7.bff b/External/SDK/Clang/Windows/Clang7.bff deleted file mode 100644 index ff33573df..000000000 --- a/External/SDK/Clang/Windows/Clang7.bff +++ /dev/null @@ -1,160 +0,0 @@ -// Clang 7.x.x -//------------------------------------------------------------------------------ -.Clang7_BasePath = '$_CURRENT_BFF_DIR_$/7.0.1' - -// Compiler -//------------------------------------------------------------------------------ -Compiler( 'Compiler-Clang7' ) -{ - .Root = '$Clang7_BasePath$' - .Executable = '$Root$\bin\clang-cl.exe' - - // Allow tests to activate some experimental behavior - #if ENABLE_RELATIVE_PATHS - .UseRelativePaths_Experimental = true - #endif - #if ENABLE_SOURCE_MAPPING - .SourceMapping_Experimental = '/fastbuild-test-mapping' - #endif -} - -// Compiler -//------------------------------------------------------------------------------ -Compiler( 'Compiler-Clang7-NonCL' ) -{ - .Root = '$Clang7_BasePath$' - .Executable = '$Root$\bin\clang.exe' - - // Allow tests to activate some experimental behavior - #if ENABLE_RELATIVE_PATHS - .UseRelativePaths_Experimental = true - #endif - #if ENABLE_SOURCE_MAPPING - .SourceMapping_Experimental = '/fastbuild-test-mapping' - #endif -} - -// ToolChain -//------------------------------------------------------------------------------ -.ToolChain_Clang_Windows_Common = -[ - // Clang for Windows relies on the VS being present: - // - crt headers - // - crt libs/dlls - Using( .ToolChain_VS_Windows_X64 ) - - .Platform = 'x64Clang' - - // Librarian - .Librarian = '$Clang7_BasePath$\bin\llvm-ar.exe' - .LibrarianOptions = 'rc "%2" "%1"' // NOTE: output must come first - - // Linker - .Linker = '$Clang7_BasePath$\bin\lld-link.exe' - .LinkerOptions = '/NODEFAULTLIB /WX /NOLOGO /INCREMENTAL:NO /OUT:"%2" "%1" /DEBUG' - + .VSLibPaths - - // File Extensions - .LibExtension = '.a' - .ExeExtension = '.exe' -] - -// ToolChain -//------------------------------------------------------------------------------ -.ToolChain_Clang_Windows = -[ - Using( .ToolChain_Clang_Windows_Common ) - - // Compiler Options - .Compiler = 'Compiler-Clang7' - .CommonCompilerOptions = ' -c' // Compile only - + ' /Z7' // Include debug info - - // Include paths - + ' -I"./"' - + .VSIncludePaths_ClangCl - - // x64 - + ' -m64' - - // Enable warnings - + ' -Wall -Werror -Wfatal-errors' // warnings as errors - + ' -Wextra' - + ' -Wshadow' - - // No RTTI - + ' /GR-' - - // Warnings that are not useful - + ' -Wno-#pragma-messages' // warning : %s [-W#pragma-messages] - + ' -Wno-c++98-compat-pedantic' // variadic macros are incompatible with C++98 - + ' -Wno-exit-time-destructors' // declaration requires an exit-time destructor - + ' -Wno-global-constructors' // declaration requires a global destructor - + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types - + ' -Wno-missing-prototypes' // no previous prototype for function '%s' - + ' -Wno-missing-variable-declarations' // no previous extern declaration for non-static variable '%s' - - // Warnings that fire but might be best to be fixed - + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier - + ' -Wno-deprecated' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor - + ' -Wno-missing-noreturn' // function '%s' could be declared with attribute 'noreturn' - + ' -Wno-old-style-cast' // use of old-style cast - + ' -Wno-switch-enum' // %u enumeration values not explicitly handled in switch: '%s', '%s'... - - .CompilerOptions = ' /TP -o"%2" "%1" $CommonCompilerOptions$' - + ' /std:c++17' // Enable c++17 features - .CompilerOptionsC = ' /TC -o"%2" "%1" $CommonCompilerOptions$' - .PCHOptions = ' /TP $CommonCompilerOptions$ "%1" /Fo"%3" /Fp"%2" /Yc"PrecompiledHeader.h"' - + ' /std:c++17' // Enable c++17 features - - // Exception Control - .UseExceptions = ' /EHs' -] - -// ToolChain -//------------------------------------------------------------------------------ -.ToolChain_ClangNonCL_Windows = -[ - Using( .ToolChain_Clang_Windows_Common ) - - // Compiler Options - .Compiler = 'Compiler-Clang7-NonCL' - .CommonCompilerOptions = ' -c' // Compile only - + ' -g' // Include debug info - - // Include paths - + ' "-I./"' - - // x64 - + ' -m64' - - // Enable warnings - + ' -Wall -Werror -Wfatal-errors' // warnings as errors - + ' -Wextra' - + ' -Wshadow' - - // No RTTI - + ' -fno-rtti' - - // Warnings that are not useful - + ' -Wno-#pragma-messages' // warning : %s [-W#pragma-messages] - + ' -Wno-c++98-compat-pedantic' // variadic macros are incompatible with C++98 - + ' -Wno-exit-time-destructors' // declaration requires an exit-time destructor - + ' -Wno-global-constructors' // declaration requires a global destructor - + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types - + ' -Wno-missing-prototypes' // no previous prototype for function '%s' - + ' -Wno-missing-variable-declarations' // no previous extern declaration for non-static variable '%s' - - // Warnings that fire but might be best to be fixed - + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier - + ' -Wno-deprecated' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor - + ' -Wno-missing-noreturn' // function '%s' could be declared with attribute 'noreturn' - + ' -Wno-old-style-cast' // use of old-style cast - + ' -Wno-switch-enum' // %u enumeration values not explicitly handled in switch: '%s', '%s'... - - .CompilerOptions = ' -x c++ -o"%2" "%1" $CommonCompilerOptions$' - + ' -std=c++17' // Enable c++17 features - .CompilerOptionsC = ' -x c -o"%2" "%1" $CommonCompilerOptions$' -] - -//------------------------------------------------------------------------------ From 775024219f4f637290dd935536da4442634c9a95 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Jul 2023 13:27:13 +0930 Subject: [PATCH 056/129] Remove remnants of support for building FASTBuild with VS pre VS2017: - we haven't needed or maintained that for a long time - this has no impact on which versions FASTBuild can use --- Code/Core/Env/Types.h | 7 ------ Code/Core/Mem/Mem.cpp | 8 +++---- Code/Core/Mem/Mem.h | 4 ++-- .../FBuild/FBuildTest/Tests/TestCache.cpp | 24 +++++-------------- 4 files changed, 12 insertions(+), 31 deletions(-) diff --git a/Code/Core/Env/Types.h b/Code/Core/Env/Types.h index 09775fa22..61e9c8e31 100644 --- a/Code/Core/Env/Types.h +++ b/Code/Core/Env/Types.h @@ -88,13 +88,6 @@ typedef signed int int32_t; #endif #endif -// Versions of Visual Studio prior to 2017 don't manage noexcept properly -#if defined( _MSC_VER ) && ( _MSC_VER < 1910 ) && !defined( __clang__ ) - #define NOEXCEPT -#else - #define NOEXCEPT noexcept -#endif - #ifndef LONGLONG typedef long long LONGLONG; #endif diff --git a/Code/Core/Mem/Mem.cpp b/Code/Core/Mem/Mem.cpp index d347d5903..bb901156a 100644 --- a/Code/Core/Mem/Mem.cpp +++ b/Code/Core/Mem/Mem.cpp @@ -139,14 +139,14 @@ void Free( void * ptr ) // Directly called void * operator new( size_t size ) { return AllocFileLine( size, "Unknown", 0 ); } void * operator new[]( size_t size ) { return AllocFileLine( size, "Unknown", 0 ); } - void operator delete( void * ptr ) NOEXCEPT { Free( ptr ); } - void operator delete[]( void * ptr ) NOEXCEPT { Free( ptr ); } + void operator delete( void * ptr ) noexcept { Free( ptr ); } + void operator delete[]( void * ptr ) noexcept { Free( ptr ); } #else #if !__has_feature( address_sanitizer ) && !__has_feature( memory_sanitizer ) && !__has_feature( thread_sanitizer ) && !defined( __SANITIZE_ADDRESS__ ) void * operator new( size_t size ) { return Alloc( size ); } void * operator new[]( size_t size ) { return Alloc( size ); } - void operator delete( void * ptr ) NOEXCEPT { Free( ptr ); } - void operator delete[]( void * ptr ) NOEXCEPT { Free( ptr ); } + void operator delete( void * ptr ) noexcept { Free( ptr ); } + void operator delete[]( void * ptr ) noexcept { Free( ptr ); } #endif #endif //------------------------------------------------------------------------------ diff --git a/Code/Core/Mem/Mem.h b/Code/Core/Mem/Mem.h index 1c3bd692a..abef3858e 100644 --- a/Code/Core/Mem/Mem.h +++ b/Code/Core/Mem/Mem.h @@ -61,8 +61,8 @@ void Free( void * ptr ); #if !__has_feature( address_sanitizer ) && !__has_feature( memory_sanitizer ) && !__has_feature( thread_sanitizer ) && !defined( __SANITIZE_ADDRESS__ ) void * operator new( size_t size ); void * operator new[]( size_t size ); -void operator delete( void * ptr ) NOEXCEPT; -void operator delete[]( void * ptr ) NOEXCEPT; +void operator delete( void * ptr ) noexcept; +void operator delete[]( void * ptr ) noexcept; #endif //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp index fb625ff9e..116f82b37 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp @@ -908,9 +908,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_Write() const TEST_ASSERT( output.Find( "warning C6201" ) && output.Find( "Index '32' is out of valid index range" ) ); TEST_ASSERT( output.Find( "warning C6386" ) && output.Find( "Buffer overrun while writing to 'buffer'" ) ); // file2.cpp - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( output.Find( "warning C6387" ) && output.Find( "could be '0'" ) ); - #endif + TEST_ASSERT( output.Find( "warning C6387" ) && output.Find( "could be '0'" ) ); // Check analysis file is present with expected errors AString xml; @@ -918,9 +916,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_Write() const TEST_ASSERT( xml.Find( "6201" ) ); TEST_ASSERT( xml.Find( "6386" ) ); LoadFileContentsAsString( mAnalyzeMSVCXMLFile2, xml ); - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( xml.Find( "6387" ) ); - #endif + TEST_ASSERT( xml.Find( "6387" ) ); } // Analyze_MSVC_WarningsOnly_Read @@ -952,9 +948,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_Read() const TEST_ASSERT( xml.Find( "6201" ) ); TEST_ASSERT( xml.Find( "6386" ) ); LoadFileContentsAsString( mAnalyzeMSVCXMLFile2, xml ); - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( xml.Find( "6387" ) ); - #endif + TEST_ASSERT( xml.Find( "6387" ) ); } // Analyze_MSVC_WarningsOnly_WriteFromDist @@ -993,9 +987,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_WriteFromDist() const TEST_ASSERT( output.Find( "warning C6201" ) && output.Find( "Index '32' is out of valid index range" ) ); TEST_ASSERT( output.Find( "warning C6386" ) && output.Find( "Buffer overrun while writing to 'buffer'" ) ); // file2.cpp - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( output.Find( "warning C6387" ) && output.Find( "could be '0': this does not adhere to the specification for the function" ) ); - #endif + TEST_ASSERT( output.Find( "warning C6387" ) && output.Find( "could be '0': this does not adhere to the specification for the function" ) ); // Check analysis file is present with expected errors AString xml; @@ -1003,9 +995,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_WriteFromDist() const TEST_ASSERT( xml.Find( "6201" ) ); TEST_ASSERT( xml.Find( "6386" ) ); LoadFileContentsAsString( mAnalyzeMSVCXMLFile2, xml ); - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( xml.Find( "6387" ) ); - #endif + TEST_ASSERT( xml.Find( "6387" ) ); } // Analyze_MSVC_WarningsOnly_ReadFromDist @@ -1046,9 +1036,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_ReadFromDist() const TEST_ASSERT( xml.Find( "6201" ) ); TEST_ASSERT( xml.Find( "6386" ) ); LoadFileContentsAsString( mAnalyzeMSVCXMLFile2, xml ); - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( xml.Find( "6387" ) ); - #endif + TEST_ASSERT( xml.Find( "6387" ) ); } // ExtraFiles From ca08179b55c6bd781a30d2972b61734b654a2b4d Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Jul 2023 13:32:29 +0930 Subject: [PATCH 057/129] Zstd: Prepare Zstd (v1.5.5) for future use --- External/Zstd/zstd-1.5.5/CHANGELOG | 800 ++ External/Zstd/zstd-1.5.5/COPYING | 339 + External/Zstd/zstd-1.5.5/LICENSE | 30 + External/Zstd/zstd-1.5.5/README.md | 223 + External/Zstd/zstd-1.5.5/lib/README.md | 224 + .../Zstd/zstd-1.5.5/lib/common/allocations.h | 55 + External/Zstd/zstd-1.5.5/lib/common/bits.h | 200 + .../Zstd/zstd-1.5.5/lib/common/bitstream.h | 437 + .../Zstd/zstd-1.5.5/lib/common/compiler.h | 358 + External/Zstd/zstd-1.5.5/lib/common/cpu.h | 213 + External/Zstd/zstd-1.5.5/lib/common/debug.c | 24 + External/Zstd/zstd-1.5.5/lib/common/debug.h | 107 + .../zstd-1.5.5/lib/common/entropy_common.c | 340 + .../zstd-1.5.5/lib/common/error_private.c | 63 + .../zstd-1.5.5/lib/common/error_private.h | 159 + External/Zstd/zstd-1.5.5/lib/common/fse.h | 639 ++ .../zstd-1.5.5/lib/common/fse_decompress.c | 311 + External/Zstd/zstd-1.5.5/lib/common/huf.h | 273 + External/Zstd/zstd-1.5.5/lib/common/mem.h | 435 + External/Zstd/zstd-1.5.5/lib/common/pool.c | 371 + External/Zstd/zstd-1.5.5/lib/common/pool.h | 90 + .../lib/common/portability_macros.h | 156 + .../Zstd/zstd-1.5.5/lib/common/threading.c | 176 + .../Zstd/zstd-1.5.5/lib/common/threading.h | 150 + External/Zstd/zstd-1.5.5/lib/common/xxhash.c | 24 + External/Zstd/zstd-1.5.5/lib/common/xxhash.h | 5686 +++++++++++++ .../Zstd/zstd-1.5.5/lib/common/zstd_common.c | 48 + .../Zstd/zstd-1.5.5/lib/common/zstd_deps.h | 111 + .../zstd-1.5.5/lib/common/zstd_internal.h | 392 + .../Zstd/zstd-1.5.5/lib/common/zstd_trace.h | 163 + .../Zstd/zstd-1.5.5/lib/compress/clevels.h | 134 + .../zstd-1.5.5/lib/compress/fse_compress.c | 624 ++ External/Zstd/zstd-1.5.5/lib/compress/hist.c | 181 + External/Zstd/zstd-1.5.5/lib/compress/hist.h | 75 + .../zstd-1.5.5/lib/compress/huf_compress.c | 1435 ++++ .../zstd-1.5.5/lib/compress/zstd_compress.c | 7032 +++++++++++++++++ .../lib/compress/zstd_compress_internal.h | 1532 ++++ .../lib/compress/zstd_compress_literals.c | 235 + .../lib/compress/zstd_compress_literals.h | 39 + .../lib/compress/zstd_compress_sequences.c | 442 ++ .../lib/compress/zstd_compress_sequences.h | 54 + .../lib/compress/zstd_compress_superblock.c | 577 ++ .../lib/compress/zstd_compress_superblock.h | 32 + .../Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h | 742 ++ .../lib/compress/zstd_double_fast.c | 758 ++ .../lib/compress/zstd_double_fast.h | 39 + .../Zstd/zstd-1.5.5/lib/compress/zstd_fast.c | 960 +++ .../Zstd/zstd-1.5.5/lib/compress/zstd_fast.h | 38 + .../Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c | 2157 +++++ .../Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h | 127 + .../Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c | 724 ++ .../Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h | 117 + .../lib/compress/zstd_ldm_geartab.h | 106 + .../Zstd/zstd-1.5.5/lib/compress/zstd_opt.c | 1472 ++++ .../Zstd/zstd-1.5.5/lib/compress/zstd_opt.h | 56 + .../zstd-1.5.5/lib/compress/zstdmt_compress.c | 1867 +++++ .../zstd-1.5.5/lib/compress/zstdmt_compress.h | 113 + .../lib/decompress/huf_decompress.c | 1882 +++++ .../lib/decompress/huf_decompress_amd64.S | 576 ++ .../zstd-1.5.5/lib/decompress/zstd_ddict.c | 244 + .../zstd-1.5.5/lib/decompress/zstd_ddict.h | 44 + .../lib/decompress/zstd_decompress.c | 2355 ++++++ .../lib/decompress/zstd_decompress_block.c | 2192 +++++ .../lib/decompress/zstd_decompress_block.h | 73 + .../lib/decompress/zstd_decompress_internal.h | 238 + External/Zstd/zstd-1.5.5/lib/zdict.h | 474 ++ External/Zstd/zstd-1.5.5/lib/zstd.h | 3020 +++++++ External/Zstd/zstd-1.5.5/lib/zstd_errors.h | 114 + 68 files changed, 45477 insertions(+) create mode 100644 External/Zstd/zstd-1.5.5/CHANGELOG create mode 100644 External/Zstd/zstd-1.5.5/COPYING create mode 100644 External/Zstd/zstd-1.5.5/LICENSE create mode 100644 External/Zstd/zstd-1.5.5/README.md create mode 100644 External/Zstd/zstd-1.5.5/lib/README.md create mode 100644 External/Zstd/zstd-1.5.5/lib/common/allocations.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/bits.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/bitstream.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/compiler.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/cpu.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/debug.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/debug.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/entropy_common.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/error_private.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/error_private.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/fse.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/fse_decompress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/huf.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/mem.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/pool.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/pool.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/portability_macros.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/threading.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/threading.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/xxhash.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/xxhash.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/zstd_common.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/zstd_deps.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/zstd_internal.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/zstd_trace.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/clevels.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/fse_compress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/hist.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/hist.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/huf_compress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_internal.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm_geartab.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.h create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress_amd64.S create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.c create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.h create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.c create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.h create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_internal.h create mode 100644 External/Zstd/zstd-1.5.5/lib/zdict.h create mode 100644 External/Zstd/zstd-1.5.5/lib/zstd.h create mode 100644 External/Zstd/zstd-1.5.5/lib/zstd_errors.h diff --git a/External/Zstd/zstd-1.5.5/CHANGELOG b/External/Zstd/zstd-1.5.5/CHANGELOG new file mode 100644 index 000000000..c7a7506ee --- /dev/null +++ b/External/Zstd/zstd-1.5.5/CHANGELOG @@ -0,0 +1,800 @@ +v1.5.5 (Apr 2023) +fix: fix rare corruption bug affecting the high compression mode, reported by @danlark1 (#3517, @terrelln) +perf: improve mid-level compression speed (#3529, #3533, #3543, @yoniko and #3552, @terrelln) +lib: deprecated bufferless block-level API (#3534) by @terrelln +cli: mmap large dictionaries to save memory, by @daniellerozenblit +cli: improve speed of --patch-from mode (~+50%) (#3545) by @daniellerozenblit +cli: improve i/o speed (~+10%) when processing lots of small files (#3479) by @felixhandte +cli: zstd no longer crashes when requested to write into write-protected directory (#3541) by @felixhandte +cli: fix decompression into block device using -o, reported by @georgmu (#3583) +build: fix zstd CLI compiled with lzma support but not zlib support (#3494) by @Hello71 +build: fix cmake does no longer require 3.18 as minimum version (#3510) by @kou +build: fix MSVC+ClangCL linking issue (#3569) by @tru +build: fix zstd-dll, version of zstd CLI that links to the dynamic library (#3496) by @yoniko +build: fix MSVC warnings (#3495) by @embg +doc: updated zstd specification to clarify corner cases, by @Cyan4973 +doc: document how to create fat binaries for macos (#3568) by @rickmark +misc: improve seekable format ingestion speed (~+100%) for very small chunk sizes (#3544) by @Cyan4973 +misc: tests/fullbench can benchmark multiple files (#3516) by @dloidolt + +v1.5.4 (Feb 2023) +perf: +20% faster huffman decompression for targets that can't compile x64 assembly (#3449, @terrelln) +perf: up to +10% faster streaming compression at levels 1-2 (#3114, @embg) +perf: +4-13% for levels 5-12 by optimizing function generation (#3295, @terrelln) +pref: +3-11% compression speed for `arm` target (#3199, #3164, #3145, #3141, #3138, @JunHe77 and #3139, #3160, @danlark1) +perf: +5-30% faster dictionary compression at levels 1-4 (#3086, #3114, #3152, @embg) +perf: +10-20% cold dict compression speed by prefetching CDict tables (#3177, @embg) +perf: +1% faster compression by removing a branch in ZSTD_fast_noDict (#3129, @felixhandte) +perf: Small compression ratio improvements in high compression mode (#2983, #3391, @Cyan4973 and #3285, #3302, @daniellerozenblit) +perf: small speed improvement by better detecting `STATIC_BMI2` for `clang` (#3080, @TocarIP) +perf: Improved streaming performance when `ZSTD_c_stableInBuffer` is set (#2974, @Cyan4973) +cli: Asynchronous I/O for improved cli speed (#2975, #2985, #3021, #3022, @yoniko) +cli: Change `zstdless` behavior to align with `zless` (#2909, @binhdvo) +cli: Keep original file if `-c` or `--stdout` is given (#3052, @dirkmueller) +cli: Keep original files when result is concatenated into a single output with `-o` (#3450, @Cyan4973) +cli: Preserve Permissions and Ownership of regular files (#3432, @felixhandte) +cli: Print zlib/lz4/lzma library versions with `-vv` (#3030, @terrelln) +cli: Print checksum value for single frame files with `-lv` (#3332, @Cyan4973) +cli: Print `dictID` when present with `-lv` (#3184, @htnhan) +cli: when `stderr` is *not* the console, disable status updates, but preserve final summary (#3458, @Cyan4973) +cli: support `--best` and `--no-name` in `gzip` compatibility mode (#3059, @dirkmueller) +cli: support for `posix` high resolution timer `clock_gettime()`, for improved benchmark accuracy (#3423, @Cyan4973) +cli: improved help/usage (`-h`, `-H`) formatting (#3094, @dirkmueller and #3385, @jonpalmisc) +cli: Fix better handling of bogus numeric values (#3268, @ctkhanhly) +cli: Fix input consists of multiple files _and_ `stdin` (#3222, @yoniko) +cli: Fix tiny files passthrough (#3215, @cgbur) +cli: Fix for `-r` on empty directory (#3027, @brailovich) +cli: Fix empty string as argument for `--output-dir-*` (#3220, @embg) +cli: Fix decompression memory usage reported by `-vv --long` (#3042, @u1f35c, and #3232, @zengyijing) +cli: Fix infinite loop when empty input is passed to trainer (#3081, @terrelln) +cli: Fix `--adapt` doesn't work when `--no-progress` is also set (#3354, @terrelln) +api: Support for Block-Level Sequence Producer (#3333, @embg) +api: Support for in-place decompression (#3432, @terrelln) +api: New `ZSTD_CCtx_setCParams()` function, set all parameters defined in a `ZSTD_compressionParameters` structure (#3403, @Cyan4973) +api: Streaming decompression detects incorrect header ID sooner (#3175, @Cyan4973) +api: Window size resizing optimization for edge case (#3345, @daniellerozenblit) +api: More accurate error codes for busy-loop scenarios (#3413, #3455, @Cyan4973) +api: Fix limit overflow in `compressBound` and `decompressBound` (#3362, #3373, Cyan4973) reported by @nigeltao +api: Deprecate several advanced experimental functions: streaming (#3408, @embg), copy (#3196, @mileshu) +bug: Fix corruption that rarely occurs in 32-bit mode with wlog=25 (#3361, @terrelln) +bug: Fix for block-splitter (#3033, @Cyan4973) +bug: Fixes for Sequence Compression API (#3023, #3040, @Cyan4973) +bug: Fix leaking thread handles on Windows (#3147, @animalize) +bug: Fix timing issues with cmake/meson builds (#3166, #3167, #3170, @Cyan4973) +build: Allow user to select legacy level for cmake (#3050, @shadchin) +build: Enable legacy support by default in cmake (#3079, @niamster) +build: Meson build script improvements (#3039, #3120, #3122, #3327, #3357, @eli-schwartz and #3276, @neheb) +build: Add aarch64 to supported architectures for zstd_trace (#3054, @ooosssososos) +build: support AIX architecture (#3219, @qiongsiwu) +build: Fix `ZSTD_LIB_MINIFY` build macro, which now reduces static library size by half (#3366, @terrelln) +build: Fix Windows issues with Multithreading translation layer (#3364, #3380, @yoniko) and ARM64 target (#3320, @cwoffenden) +build: Fix `cmake` script (#3382, #3392, @terrelln and #3252 @Tachi107 and #3167 @Cyan4973) +doc: Updated man page, providing more details for `--train` mode (#3112, @Cyan4973) +doc: Add decompressor errata document (#3092, @terrelln) +misc: Enable Intel CET (#2992, #2994, @hjl-tools) +misc: Fix `contrib/` seekable format (#3058, @yhoogstrate and #3346, @daniellerozenblit) +misc: Improve speed of the one-file library generator (#3241, @wahern and #3005, @cwoffenden) + +v1.5.3 (dev version, unpublished) + +v1.5.2 (Jan, 2022) +perf: Regain Minimal memset()-ing During Reuse of Compression Contexts (@Cyan4973, #2969) +build: Build Zstd with `noexecstack` on All Architectures (@felixhandte, #2964) +doc: Clarify Licensing (@terrelln, #2981) + +v1.5.1 (Dec, 2021) +perf: rebalanced compression levels, to better match the intended speed/level curve, by @senhuang42 +perf: faster huffman decoder, using x64 assembly, by @terrelln +perf: slightly faster high speed modes (strategies fast & dfast), by @felixhandte +perf: improved binary size and faster compilation times, by @terrelln +perf: new row64 mode, used notably in level 12, by @senhuang42 +perf: faster mid-level compression speed in presence of highly repetitive patterns, by @senhuang42 +perf: minor compression ratio improvements for small data at high levels, by @cyan4973 +perf: reduced stack usage (mostly useful for Linux Kernel), by @terrelln +perf: faster compression speed on incompressible data, by @bindhvo +perf: on-demand reduced ZSTD_DCtx state size, using build macro ZSTD_DECODER_INTERNAL_BUFFER, at a small cost of performance, by @bindhvo +build: allows hiding static symbols in the dynamic library, using build macro, by @skitt +build: support for m68k (Motorola 68000's), by @cyan4973 +build: improved AIX support, by @Helflym +build: improved meson unofficial build, by @eli-schwartz +cli : custom memory limit when training dictionary (#2925), by @embg +cli : report advanced parameters information when compressing in very verbose mode (``-vv`), by @Svetlitski-FB + +v1.5.0 (May 11, 2021) +api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42) + `ZSTD_defaultCLevel()` + `ZSTD_getDictID_fromCDict()` +api: Several experimental functions have been deprecated and will emit a compiler warning (#2582, @senhuang42) + `ZSTD_compress_advanced()` + `ZSTD_compress_usingCDict_advanced()` + `ZSTD_compressBegin_advanced()` + `ZSTD_compressBegin_usingCDict_advanced()` + `ZSTD_initCStream_srcSize()` + `ZSTD_initCStream_usingDict()` + `ZSTD_initCStream_usingCDict()` + `ZSTD_initCStream_advanced()` + `ZSTD_initCStream_usingCDict_advanced()` + `ZSTD_resetCStream()` +api: ZSTDMT_NBWORKERS_MAX reduced to 64 for 32-bit environments (@Cyan4973) +perf: Significant speed improvements for middle compression levels (#2494, @senhuang42 @terrelln) +perf: Block splitter to improve compression ratio, enabled by default for high compression levels (#2447, @senhuang42) +perf: Decompression loop refactor, speed improvements on `clang` and for `--long` modes (#2614 #2630, @Cyan4973) +perf: Reduced stack usage during compression and decompression entropy stage (#2522 #2524, @terrelln) +bug: Improve setting permissions of created files (#2525, @felixhandte) +bug: Fix large dictionary non-determinism (#2607, @terrelln) +bug: Fix non-determinism test failures on Linux i686 (#2606, @terrelln) +bug: Fix various dedicated dictionary search bugs (#2540 #2586, @senhuang42 @felixhandte) +bug: Ensure `ZSTD_estimateCCtxSize*() `monotonically increases with compression level (#2538, @senhuang42) +bug: Fix --patch-from mode parameter bound bug with small files (#2637, @occivink) +bug: Fix UBSAN error in decompression (#2625, @terrelln) +bug: Fix superblock compression divide by zero bug (#2592, @senhuang42) +bug: Make the number of physical CPU cores detection more robust (#2517, @PaulBone) +doc: Improve `zdict.h` dictionary training API documentation (#2622, @terrelln) +doc: Note that public `ZSTD_free*()` functions accept NULL pointers (#2521, @animalize) +doc: Add style guide docs for open source contributors (#2626, @Cyan4973) +tests: Better regression test coverage for different dictionary modes (#2559, @senhuang42) +tests: Better test coverage of index reduction (#2603, @terrelln) +tests: OSS-Fuzz coverage for seekable format (#2617, @senhuang42) +tests: Test coverage for ZSTD threadpool API (#2604, @senhuang42) +build: Dynamic library built multithreaded by default (#2584, @senhuang42) +build: Move `zstd_errors.h` and `zdict.h` to `lib/` root (#2597, @terrelln) +build: Allow `ZSTDMT_JOBSIZE_MIN` to be configured at compile-time, reduce default to 512KB (#2611, @Cyan4973) +build: Single file library build script moved to `build/` directory (#2618, @felixhandte) +build: `ZBUFF_*()` is no longer built by default (#2583, @senhuang42) +build: Fixed Meson build (#2548, @SupervisedThinking @kloczek) +build: Fix excessive compiler warnings with clang-cl and CMake (#2600, @nickhutchinson) +build: Detect presence of `md5` on Darwin (#2609, @felixhandte) +build: Avoid SIGBUS on armv6 (#2633, @bmwiedmann) +cli: `--progress` flag added to always display progress bar (#2595, @senhuang42) +cli: Allow reading from block devices with `--force` (#2613, @felixhandte) +cli: Fix CLI filesize display bug (#2550, @Cyan4973) +cli: Fix windows CLI `--filelist` end-of-line bug (#2620, @Cyan4973) +contrib: Various fixes for linux kernel patch (#2539, @terrelln) +contrib: Seekable format - Decompression hanging edge case fix (#2516, @senhuang42) +contrib: Seekable format - New seek table-only API (#2113 #2518, @mdittmer @Cyan4973) +contrib: Seekable format - Fix seek table descriptor check when loading (#2534, @foxeng) +contrib: Seekable format - Decompression fix for large offsets, (#2594, @azat) +misc: Automatically published release tarballs available on Github (#2535, @felixhandte) + +v1.4.9 (Mar 1, 2021) +bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte) +bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln) +api: Add (De)Compression Tracing Functionality (#2482, @terrelln) +api: Support References to Multiple DDicts (#2446, @senhuang42) +api: Add Function to Generate Skippable Frame (#2439, @senhuang42) +perf: New Algorithms for the Long Distance Matcher (#2483, @mpu) +perf: Performance Improvements for Long Distance Matcher (#2464, @mpu) +perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln) +cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte) +cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte) +cli: Improve Help Message (#2500, @senhuang42) +tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973) +tests: Correctly Invoke md5 Utility on NetBSD (#2492, @niacat) +tests: Avoid Using `stat -c` on NetBSD (#2513, @felixhandte) +build: Zstd CLI Can Now be Linked to Dynamic `libzstd` (#2457, #2454 @Cyan4973) +build: Hide and Avoid Using Static-Only Symbols (#2501, #2504, @skitt) +build: CMake: Enable Only C for lib/ and programs/ Projects (#2498, @concatime) +build: CMake: Use `configure_file()` to Create the `.pc` File (#2462, @lazka) +build: Fix Fuzzer Compiler Detection & Update UBSAN Flags (#2503, @terrelln) +build: Add Guards for `_LARGEFILE_SOURCE` and `_LARGEFILE64_SOURCE` (#2444, @indygreg) +build: Improve `zlibwrapper` Makefile (#2437, @Cyan4973) +contrib: Add `recover_directory` Program (#2473, @terrelln) +doc: Change License Year to 2021 (#2452 & #2465, @terrelln & @senhuang42) +doc: Fix Typos (#2459, @ThomasWaldmann) + +v1.4.8 (Dec 18, 2020) +hotfix: wrong alignment of an internal buffer + +v1.4.7 (Dec 16, 2020) +perf: stronger --long mode at high compression levels, by @senhuang42 +perf: stronger --patch-from at high compression levels, thanks to --long improvements +perf: faster dictionary compression at medium compression levels, by @felixhandte +perf: small speed & memory usage improvements for ZSTD_compress2(), by @terrelln +perf: improved fast compression speeds with Visual Studio, by @animalize +cli : Set nb of threads with environment variable ZSTD_NBTHREADS, by @senhuang42 +cli : accept decompressing files with *.zstd suffix +cli : provide a condensed summary by default when processing multiple files +cli : fix : stdin input no longer confused as user prompt +cli : improve accuracy of several error messages +api : new sequence ingestion API, by @senhuang42 +api : shared thread pool: control total nb of threads used by multiple compression jobs, by @marxin +api : new ZSTD_getDictID_fromCDict(), by @LuAPi +api : zlibWrapper only uses public API, and is compatible with dynamic library, by @terrelln +api : fix : multithreaded compression has predictable output even in special cases (see #2327) (issue not accessible from cli) +api : fix : dictionary compression correctly respects dictionary compression level (see #2303) (issue not accessible from cli) +build: fix cmake script when using path with spaces, by @terrelln +build: improved compile-time detection of aarch64/neon platforms, by @bsdimp +build: Fix building on AIX 5.1, by @likema +build: compile paramgrill with cmake on Windows, requested by @mirh +doc : clarify repcode updates in format specification, by @felixhandte + +v1.4.6 +fix : Always return dstSize_tooSmall when that is the case +fix : Fix ZSTD_initCStream_advanced() with static allocation and no dictionary +perf: Improve small block decompression speed by 20%+, by @terrelln +perf: Reduce compression stack usage by 1 KB, by @terrelln +perf: Improve decompression speed by improving ZSTD_wildcopy, by @helloguo (#2252, #2256) +perf: Improve histogram construction, by @cyan4973 (#2253) +cli : Add --output-dir-mirror option, by @xxie24 (#2219) +cli : Warn when (de)compressing multiple files into a single output, by @senhuang42 (#2279) +cli : Improved progress bar and status summary when (de)compressing multiple files, by @senhuang42 (#2283) +cli : Call stat less often, by @felixhandte (#2262) +cli : Allow --patch-from XXX and --filelist XXX in addition to --patch-from=XXX and --filelist=XXX, by @cyan4973 (#2250) +cli : Allow --patch-from to compress stdin with --stream-size, by @bimbashrestha (#2206) +api : Do not install zbuff.h, since it has long been deprecated, by @cyan4973 (#2166). +api : Fix ZSTD_CCtx_setParameter() with ZSTD_c_compressionLevel to make 0 mean default level, by @i-do-cpp (#2291) +api : Rename ZSTDMT_NBTHREADS_MAX to ZSTDMT_NBWORKERS_MAX, by @marxin (#2228). +build: Install pkg-config file with CMake and MinGW, by @tonytheodore (#2183) +build: Install DLL with CMake on Windows, by @BioDataAnalysis (#2221) +build: Fix DLL install location with CMake, by @xantares and @bimbashrestha (#2186) +build: Add ZSTD_NO_UNUSED_FUNCTIONS macro to hide unused functions +build: Add ZSTD_NO_INTRINSICS macro to avoid explicit intrinsics +build: Add STATIC_BMI2 macro for compile time detection of BMI2 on MSVC, by @Niadb (#2258) +build: Fix -Wcomma warnings, by @cwoffenden +build: Remove distutils requirement for meson build, by @neheb (#2197) +build: Fix cli compilation with uclibc +build: Fix cli compilation without st_mtime, by @ffontaine (#2246) +build: Fix shadowing warnings in library +build: Fix single file library compilation with Enscripten, by @yoshihitoh (#2227) +misc: Improve single file library and include dictBuilder, by @cwoffenden +misc: Allow compression dictionaries with missing symbols +misc: Add freestanding translation script in contrib/freestanding_lib +misc: Collect all of zstd's libc dependencies into zstd_deps.h +doc : Add ZSTD_versionString() to manual, by @animalize +doc : Fix documentation for ZSTD_CCtxParams_setParameter(), by @felixhandte (#2270) + +v1.4.5 (May 22, 2020) +fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln +perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln +perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta) +perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh) +perf: Small level 1 compression speed gains (depending on compiler) +cli : New --patch-from command, create and apply patches from files, by @bimbashreshta +cli : New --filelist= : Provide a list of files to operate upon from a file +cli : -b -d command can now benchmark decompression on multiple files +cli : New --no-content-size command +cli : New --show-default-cparams information command +api : ZDICT_finalizeDictionary() is promoted to stable (#2111) +api : new experimental parameter ZSTD_d_stableOutBuffer (#2094) +build: Generate a single-file libzstd library (#2065, by @cwoffenden) +build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte) +build: zstd now compiles cleanly under -pedantic (#2099) +build: zstd now compiles with make-4.3 +build: Support mingw cross-compilation from Linux, by @Ericson2314 +build: Meson multi-thread build fix on windows +build: Some misc icc fixes backed by new ci test on travis +misc: bitflip analyzer tool, by @felixhandte +misc: Extend largeNbDicts benchmark to compression +misc: Edit-distance match finder in contrib/ +doc : Improved beginner CONTRIBUTING.md docs +doc : New issue templates for zstd + +v1.4.4 (Nov 6, 2019) +perf: Improved decompression speed, by > 10%, by @terrelln +perf: Better compression speed when re-using a context, by @felixhandte +perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42 +perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha +perf: minor generic speed optimization, by @davidbolvansky +api: new ability to extract sequences from the parser for analysis, by @bimbashrestha +api: fixed decoding of magic-less frames, by @terrelln +api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK +cli: Named pipes support, by @bimbashrestha +cli: short tar's extension support, by @stokito +cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42 +cli: commands --stream-size=# and --size-hint=#, by @nmagerko +cli: command --exclude-compressed, by @shashank0791 +cli: faster `-t` test mode +cli: improved some error messages, by @vangyzen +cli: fix command `-D dictionary` on Windows, reported by @artyompetrov +cli: fix rare deadlock condition within dictionary builder, by @terrelln +build: single-file decoder with emscripten compilation script, by @cwoffenden +build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive +build: fixed deprecation warning for certain gcc version, reported by @jasonma163 +build: fix compilation on old gcc versions, by @cemeyer +build: improved installation directories for cmake script, by Dmitri Shubin +pack: modified pkgconfig, for better integration into openwrt, requested by @neheb +misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format +misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro + +v1.4.3 (Aug 20, 2019) +bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709) +bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722) +build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705) + +v1.4.2 (Jul 26, 2019) +bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696) +bug: Fix seekable decompression in-memory API by @iburinoc (#1695) +misc: Validate blocks are smaller than size limit by @vivekmg (#1685) +misc: Restructure source files by @ephiepark (#1679) + +v1.4.1 (Jul 20, 2019) +bug: Fix data corruption in niche use cases by @terrelln (#1659) +bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595) +bug: Fix out of bounds read by @terrelln (#1590) +perf: Improve decode speed by ~7% @mgrice (#1668) +perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681) +perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658) +perf: Improve compression ratio for small windowLog by @cyan4973 (#1624) +perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635) +api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656) +cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640) +cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631) +cli: Restrict read permissions on destination files by @chungy (#1644) +cli: zstdgrep: handle -f flag by @felixhandte (#1618) +cli: zstdcat: follow symlinks by @vejnar (#1604) +doc: Remove extra size limit on compressed blocks by @felixhandte (#1689) +doc: Fix typo by @yk-tanigawa (#1633) +doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629) +build: CMake: support building with LZ4 @leeyoung624 (#1626) +build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647) +build: CMake: respect existing uninstall target by @j301scott (#1619) +build: Make: skip multithread tests when built without support by @michaelforney (#1620) +build: Make: Fix examples/ test target by @sjnam (#1603) +build: Meson: rename options out of deprecated namespace by @lzutao (#1665) +build: Meson: fix build by @lzutao (#1602) +build: Visual Studio: don't export symbols in static lib by @scharan (#1650) +build: Visual Studio: fix linking by @absotively (#1639) +build: Fix MinGW-W64 build by @myzhang1029 (#1600) +misc: Expand decodecorpus coverage by @ephiepark (#1664) + +v1.4.0 (Apr 17, 2019) +perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln +api: Move the advanced API, including all functions in the staging section, to the stable section +api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress +api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter +api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter +api: Don't export ZSTDMT functions from the shared library by default +api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT +api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao +api: Fix ZSTD_decompressDCtx() corner cases with a dictionary +api: Move ZSTD_getDictID_*() functions to the stable section +api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln +api: Allow compression parameters to be set when a dictionary is used +api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called +api: Fix ZSTD_estimateCStreamSize_usingCCtxParams() +api: Setting ZSTD_d_maxWindowLog to 0 means use the default +cli: Ensure that a dictionary is not used to compress itself by @shakeelrao +cli: Add --[no-]compress-literals flag to enable or disable literal compression +doc: Update the examples to use the advanced API +doc: Explain how to transition from old streaming functions to the advanced API in the header +build: Improve the Windows release packages +build: Improve CMake build by @hjmjohnson +build: Build fixes for FreeBSD by @lwhsu +build: Remove redundant warnings by @thatsafunnyname +build: Fix tests on OpenBSD by @bket +build: Extend fuzzer build system to work with the new clang engine +build: CMake now creates the libzstd.so.1 symlink +build: Improve Menson build by @lzutao +misc: Fix symbolic link detection on FreeBSD +misc: Use physical core count for -T0 on FreeBSD by @cemeyer +misc: Fix zstd --list on truncated files by @kostmo +misc: Improve logging in debug mode by @felixhandte +misc: Add CirrusCI tests by @lwhsu +misc: Optimize dictionary memory usage in corner cases +misc: Improve the dictionary builder on small or homogeneous data +misc: Fix spelling across the repo by @jsoref + +v1.3.8 (Dec 28, 2018) +perf: better decompression speed on large files (+7%) and cold dictionaries (+15%) +perf: slightly better compression ratio at high compression modes +api : finalized advanced API, last stage before "stable" status +api : new --rsyncable mode, by @terrelln +api : support decompression of empty frames into NULL (used to be an error) (#1385) +build: new set of macros to build a minimal size decoder, by @felixhandte +build: fix compilation on MIPS32, reported by @clbr (#1441) +build: fix compilation with multiple -arch flags, by @ryandesign +build: highly upgraded meson build, by @lzutao +build: improved buck support, by @obelisk +build: fix cmake script : can create debug build, by @pitrou +build: Makefile : grep works on both colored consoles and systems without color support +build: fixed zstd-pgo, by @bmwiedemann +cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423) +cli : --no-progress flag, preserving final summary (#1371), by @terrelln +cli : ensure destination file is not source file (#1422) +cli : clearer error messages, especially when input file not present +doc : clarified zstd_compression_format.md, by @ulikunitz +misc: fixed zstdgrep, returns 1 on failure, by @lzutao +misc: NEWS renamed as CHANGELOG, in accordance with fboss + +v1.3.7 (Oct 20, 2018) +perf: slightly better decompression speed on clang (depending on hardware target) +fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10 +build: no longer build backtrace by default in release mode; restrict further automatic mode +build: control backtrace support through build macro BACKTRACE +misc: added man pages for zstdless and zstdgrep, by @samrussell + +v1.3.6 (Oct 6, 2018) +perf: much faster dictionary builder, by @jenniferliu +perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte +perf: faster dictionary decompression when using a very large number of dictionaries simultaneously +cli : fix : does no longer overwrite destination when source does not exist (#1082) +cli : new command --adapt, for automatic compression level adaptation +api : fix : block api can be streamed with > 4 GB, reported by @catid +api : reduced ZSTD_DDict size by 2 KB +api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel(). +build: support Haiku target, by @korli +build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT. +doc : zstd_compression_format.md updated to match wording in IETF RFC 8478 +misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97 + +v1.3.5 (Jun 29, 2018) +perf: much faster dictionary compression, by @felixhandte +perf: small quality improvement for dictionary generation, by @terrelln +perf: slightly improved high compression levels (notably level 19) +mem : automatic memory release for long duration contexts +cli : fix : overlapLog can be manually set +cli : fix : decoding invalid lz4 frames +api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln +api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln +build: select custom libzstd scope through control macros, by @GeorgeLu97 +build: OpenBSD patch, by @bket +build: make and make all are compatible with -j +doc : clarify zstd_compression_format.md, updated for IETF RFC process +misc: pzstd compatible with reproducible compilation, by @lamby + +v1.3.4 (Mar 27, 2018) +perf: faster speed (especially decoding speed) on recent cpus (haswell+) +perf: much better performance associating --long with multi-threading, by @terrelln +perf: better compression at levels 13-15 +cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior) +cli : smoother status report in multi-threading mode +cli : added command --fast=#, for faster compression modes +cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb) +api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode +api : compression levels can be negative, for even more speed +api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime +api : ZSTDMT can accept new compression parameters during compression +api : implemented all advanced dictionary decompression prototypes +build: improved meson recipe, by Shawn Landden (@shawnl) +build: VS2017 scripts, by @HaydnTrigg +misc: all /contrib projects fixed +misc: added /contrib/docker script by @gyscos + +v1.3.3 (Dec 21, 2017) +perf: faster zstd_opt strategy (levels 16-19) +fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen +cli : fix : content size written in header by default +cli : fix : improved LZ4 format support, by @felixhandte +cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file +api : fix : support large skippable frames, by @terrelln +api : fix : streaming interface was adding a useless 3-bytes null block to small frames +api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown" +build: fix : compilation under rhel6 and centos6, reported by @pixelb +build: added `check` target + +v1.3.2 (Oct 10, 2017) +new : long range mode, using --long command, by Stella Lau (@stellamplau) +new : ability to generate and decode magicless frames (#591) +changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode +fix : multi-threading compression works with custom allocators +fix : ZSTD_sizeof_CStream() was over-evaluating memory usage +fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22) +fix : 32-bits build can now decode large offsets (levels 21+) +cli : added LZ4 frame support by default, by Felix Handte (@felixhandte) +cli : improved --list output +cli : new : can split input file for dictionary training, using command -B# +cli : new : clean operation artefact on Ctrl-C interruption +cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851) +cli : fix : write file size in header in multiple-files mode +api : added macro ZSTD_COMPRESSBOUND() for static allocation +api : experimental : new advanced decompression API +api : fix : sizeof_CCtx() used to over-estimate +build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819) +build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818) +example : added streaming_memory_usage +license : changed /examples license to BSD + GPLv2 +license : fix a few header files to reflect new license (#825) + +v1.3.1 (Aug 21, 2017) +New license : BSD + GPLv2 +perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk) +perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760) +cli : improved and fixed --list command, by @ib (#772) +cli : command -vV to list supported formats, by @ib (#771) +build : fixed binary variants, reported by @svenha (#788) +build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718) +API exp : breaking change : ZSTD_getframeHeader() provides more information +API exp : breaking change : pinned down values of error codes +doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz) +new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74) +new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau) +updated : contrib/linux-kernel, by Nick Terrell (@terrelln) + +v1.3.0 (Jul 6, 2017) +cli : new : `--list` command, by Paul Cruz +cli : changed : xz/lzma support enabled by default +cli : changed : `-t *` continue processing list after a decompression error +API : added : ZSTD_versionString() +API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell +API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter() +API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx() +API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700) +API exp : clarified memory estimation / measurement functions. +API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1 +tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz +new : contrib/seekable_format, demo and API, by Sean Purcell +changed : contrib/linux-kernel, updated version and license, by Nick Terrell + +v1.2.0 (May 5, 2017) +cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable) +cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell +cli : new : zstdmt symlink hardwired to `zstd -T0` +cli : new : command --threads=# (#671) +cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell +cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters +cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell +cli : fix : does not output compressed data on console +cli : fix : ignore symbolic links unless --force specified, +API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument +API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters. +API : improved: ZSTDMT_compressCCtx() reduced memory usage +API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634) +API : fix : src size stored in frame header is controlled at end of frame +API : fix : enforced consistent rules for pledgedSrcSize==0 (#641) +API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate +build: improved cmake script, by @Majlen +build: enabled Multi-threading support for *BSD, by Baptiste Daroussin +tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target. +new : contrib/linux-kernel version, by Nick Terrell + +v1.1.4 (Mar 18, 2017) +cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski +cli : new : advanced benchmark command --priority=rt +cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 +cli : fix : --rm remains silent when input is stdin +cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski +speed : improved decompression speed in streaming mode for single shot scenarios (+5%) +memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB +arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell +API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize() +API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value) +build : new: meson build system in contrib/meson, by Dima Krasner +build : improved cmake script, by @Majlen +build : added -Wformat-security flag, as recommended by Padraig Brady +doc : new : educational decoder, by Sean Purcell + +v1.1.3 (Feb 7, 2017) +cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) +cli : new : experimental target `make zstdmt`, with multi-threading support +cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano. +cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski +cli : fix zstdless on Mac OS-X, by Andrew Janke +cli : fix #232 "compress non-files" +dictBuilder : improved dictionary generation quality, thanks to Nick Terrell +API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental) +API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul +API : new : ZDICT_finalizeDictionary() +API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511) +API : fix : all symbols properly exposed in libzstd, by Nick Terrell +build : support for Solaris target, by Przemyslaw Skibinski +doc : clarified specification, by Sean Purcell + +v1.1.2 (Dec 15, 2016) +API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init +API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize() +API : zbuff : changed : prototypes now generate deprecation warnings +lib : improved : faster decompression speed at ultra compression settings and 32-bits mode +lib : changed : only public ZSTD_ symbols are now exposed +lib : changed : reduced usage of stack memory +lib : fixed : several corner case bugs, by Nick Terrell +cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski +cli : new : preserve file attributes +cli : new : added zstdless and zstdgrep tools +cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) +cli : fixed : zstdcat +zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski +install : better compatibility with FreeBSD, by Dimitry Andric +source tree : changed : zbuff source files moved to lib/deprecated + +v1.1.1 (Nov 2, 2016) +New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption +New : doc/zstd_manual.html, by Przemyslaw Skibinski +Improved : slightly better compression ratio at --ultra levels (>= 20) +Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report +Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section) +Added : example/multiple_streaming_compression.c +Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h) +Updated man page +Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets + +v1.1.0 (Sep 28, 2016) +New : contrib/pzstd, parallel version of zstd, by Nick Terrell +added : NetBSD install target (#338) +Improved : speed for batches of small files +Improved : speed of zlib wrapper, by Przemyslaw Skibinski +Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier +Fixed : CLI -d output to stdout by default when input is stdin (#322) +Fixed : CLI correctly detects console on Mac OS-X +Fixed : CLI supports recursive mode `-r` on Mac OS-X +Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski +Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) +Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) +Fixed : zstd-pgo, reported by octoploid (#329) + +v1.0.0 (Sep 1, 2016) +Change Licensing, all project is now BSD, Copyright Facebook +Small decompression speed improvement +API : Streaming API supports legacy format +API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter() +CLI supports legacy formats v0.4+ +Fixed : compression fails on certain huge files, reported by Jesse McGrew +Enhanced documentation, by Przemyslaw Skibinski + +v0.8.1 (Aug 18, 2016) +New streaming API +Changed : --ultra now enables levels beyond 19 +Changed : -i# now selects benchmark time in second +Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell +Fixed : speed regression on specific patterns (#272) +Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) +Fixed : ICC compilation, by Przemyslaw Skibinski + +v0.8.0 (Aug 2, 2016) +Improved : better speed on clang and gcc -O2, thanks to Eric Biggers +New : Build on FreeBSD and DragonFly, thanks to JrMarino +Changed : modified API : ZSTD_compressEnd() +Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist +Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers +Fixed : large dictionaries (> 384 KB), reported by Ilona Papava +Fixed : checksum correctly checked in single-pass mode +Fixed : combined --test amd --rm, reported by Andreas M. Nilsson +Modified : minor compression level adaptations +Updated : compression format specification to v0.2.0 +changed : zstd.h moved to /lib directory + +v0.7.5 (Aug 1, 2016) +Transition version, supporting decoding of v0.8.x + +v0.7.4 (Jul 17, 2016) +Added : homebrew for Mac, by Daniel Cade +Added : more examples +Fixed : segfault when using small dictionaries, reported by Felix Handte +Modified : default compression level for CLI is now 3 +Updated : specification, to v0.1.1 + +v0.7.3 (Jul 9, 2016) +New : compression format specification +New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. +New : `ZSTD_getDecompressedSize()` +New : OpenBSD target, by Juan Francisco Cantero Hurtado +New : `examples` directory +fixed : dictBuilder using HC levels, reported by Bartosz Taudul +fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte +fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski +modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) +modified : legacy functions no longer need magic number + +v0.7.2 (Jul 4, 2016) +fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. +fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. +fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner. + +v0.7.1 (Jun 23, 2016) +fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier +fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв +fixed : corruption issue, reported by cj +modified : checksum enabled by default in command line mode + +v0.7.0 (Jun 17, 2016) +New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski +New : Command `--rm`, to remove source file after successful de/compression +New : Visual build scripts, by Christophe Chevalier +New : Support for Sparse File-systems (do not use space for zero-filled sectors) +New : Frame checksum support +New : Support pass-through mode (when using `-df`) +API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()` +API : create dictionary files from custom content, by Giuseppe Ottaviano +API : support for custom malloc/free functions +New : controllable Dictionary ID +New : Support for skippable frames + +v0.6.1 (May 13, 2016) +New : zlib wrapper API, thanks to Przemyslaw Skibinski +New : Ability to compile compressor / decompressor separately +Changed : new lib directory structure +Fixed : Legacy codec v0.5 compatible with dictionary decompression +Fixed : Decoder corruption error (#173) +Fixed : null-string roundtrip (#176) +New : benchmark mode can select directory as input +Experimental : midipix support, VMS support + +v0.6.0 (Apr 13, 2016) +Stronger high compression modes, thanks to Przemyslaw Skibinski +API : ZSTD_getFrameParams() provides size of decompressed content +New : highest compression modes require `--ultra` command to fully unleash their capacity +Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner + +v0.5.1 (Feb 18, 2016) +New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski +Changed : Dictionary builder integrated into libzstd and zstd cli +Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`. +Fix : high compression modes for big-endian platforms +New : zstd cli : `-t` | `--test` command + +v0.5.0 (Feb 5, 2016) +New : dictionary builder utility +Changed : streaming & dictionary API +Improved : better compression of small data + +v0.4.7 (Jan 22, 2016) +Improved : small compression speed improvement in HC mode +Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default +fix : bt search bug + +v0.4.6 (Jan 13, 2016) +fix : fast compression mode on Windows +New : cmake configuration file, thanks to Artyom Dymchenko +Improved : high compression mode on repetitive data +New : block-level API +New : ZSTD_duplicateCCtx() + +v0.4.5 (Dec 18, 2015) +new : -m/--multiple : compress/decompress multiple files + +v0.4.4 (Dec 14, 2015) +Fixed : high compression modes for Windows 32 bits +new : external dictionary API extended to buffered mode and accessible through command line +new : windows DLL project, thanks to Christophe Chevalier + +v0.4.3 (Dec 7, 2015) +new : external dictionary API +new : zstd-frugal + +v0.4.2 (Dec 2, 2015) +Generic minor improvements for small blocks +Fixed : big-endian compatibility, by Peter Harris (#85) + +v0.4.1 (Dec 1, 2015) +Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) +removed `zstd.c` + +v0.4.0 (Nov 29, 2015) +Command line utility compatible with high compression levels +Removed zstdhc => merged into zstd +Added : ZBUFF API (see zstd_buffered.h) +Rolling buffer support + +v0.3.6 (Nov 10, 2015) +small blocks params + +v0.3.5 (Nov 9, 2015) +minor generic compression improvements + +v0.3.4 (Nov 6, 2015) +Faster fast cLevels + +v0.3.3 (Nov 5, 2015) +Small compression ratio improvement + +v0.3.2 (Nov 2, 2015) +Fixed Visual Studio + +v0.3.1 (Nov 2, 2015) +Small compression ratio improvement + +v0.3 (Oct 30, 2015) +HC mode : compression levels 2-26 + +v0.2.2 (Oct 28, 2015) +Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier + +v0.2.1 (Oct 24, 2015) +Fix : Read errors, advanced fuzzer tests, by Hanno Böck + +v0.2.0 (Oct 22, 2015) +**Breaking format change** +Faster decompression speed +Can still decode v0.1 format + +v0.1.3 (Oct 15, 2015) +fix uninitialization warning, reported by Evan Nemerson + +v0.1.2 (Sep 11, 2015) +frame concatenation support + +v0.1.1 (Aug 27, 2015) +fix compression bug +detects write-flush errors + +v0.1.0 (Aug 25, 2015) +first release diff --git a/External/Zstd/zstd-1.5.5/COPYING b/External/Zstd/zstd-1.5.5/COPYING new file mode 100644 index 000000000..ecbc05937 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. \ No newline at end of file diff --git a/External/Zstd/zstd-1.5.5/LICENSE b/External/Zstd/zstd-1.5.5/LICENSE new file mode 100644 index 000000000..75800288c --- /dev/null +++ b/External/Zstd/zstd-1.5.5/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/External/Zstd/zstd-1.5.5/README.md b/External/Zstd/zstd-1.5.5/README.md new file mode 100644 index 000000000..f91e68fdb --- /dev/null +++ b/External/Zstd/zstd-1.5.5/README.md @@ -0,0 +1,223 @@ +

Zstandard

+ +__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm, +targeting real-time compression scenarios at zlib-level and better compression ratios. +It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy). + +Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available. +This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library, +and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files. +Should your project require another programming language, +a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages). + +**Development branch status:** + +[![Build Status][travisDevBadge]][travisLink] +[![Build status][CircleDevBadge]][CircleLink] +[![Build status][CirrusDevBadge]][CirrusLink] +[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink] + +[travisDevBadge]: https://api.travis-ci.com/facebook/zstd.svg?branch=dev "Continuous Integration test suite" +[travisLink]: https://travis-ci.com/facebook/zstd +[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite" +[CircleLink]: https://circleci.com/gh/facebook/zstd +[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev +[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd +[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg +[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd + +## Benchmarks + +For reference, several fast compression algorithms were tested and compared +on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`), +with a Core i7-9700K CPU @ 4.9GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with [gcc] 9.3.0, +on the [Silesia compression corpus]. + +[lzbench]: https://github.com/inikep/lzbench +[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia +[gcc]: https://gcc.gnu.org/ + +| Compressor name | Ratio | Compression| Decompress.| +| --------------- | ------| -----------| ---------- | +| **zstd 1.5.1 -1** | 2.887 | 530 MB/s | 1700 MB/s | +| [zlib] 1.2.11 -1 | 2.743 | 95 MB/s | 400 MB/s | +| brotli 1.0.9 -0 | 2.702 | 395 MB/s | 450 MB/s | +| **zstd 1.5.1 --fast=1** | 2.437 | 600 MB/s | 2150 MB/s | +| **zstd 1.5.1 --fast=3** | 2.239 | 670 MB/s | 2250 MB/s | +| quicklz 1.5.0 -1 | 2.238 | 540 MB/s | 760 MB/s | +| **zstd 1.5.1 --fast=4** | 2.148 | 710 MB/s | 2300 MB/s | +| lzo1x 2.10 -1 | 2.106 | 660 MB/s | 845 MB/s | +| [lz4] 1.9.3 | 2.101 | 740 MB/s | 4500 MB/s | +| lzf 3.6 -1 | 2.077 | 410 MB/s | 830 MB/s | +| snappy 1.1.9 | 2.073 | 550 MB/s | 1750 MB/s | + +[zlib]: https://www.zlib.net/ +[lz4]: https://lz4.github.io/lz4/ + +The negative compression levels, specified with `--fast=#`, +offer faster compression and decompression speed +at the cost of compression ratio (compared to level 1). + +Zstd can also offer stronger compression ratios at the cost of compression speed. +Speed vs Compression trade-off is configurable by small increments. +Decompression speed is preserved and remains roughly the same at all settings, +a property shared by most LZ compression algorithms, such as [zlib] or lzma. + +The following tests were run +on a server running Linux Debian (`Linux version 4.14.0-3-amd64`) +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with [gcc] 7.3.0, +on the [Silesia compression corpus]. + +Compression Speed vs Ratio | Decompression Speed +---------------------------|-------------------- +![Compression Speed vs Ratio](doc/images/CSpeed2.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/DSpeed3.png "Decompression Speed") + +A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph. +For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png). + + +## The case for Small Data compression + +Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. + +The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon. + +To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data. +Training Zstandard is achieved by providing it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression. +Using this dictionary, the compression ratio achievable on small data improves dramatically. + +The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users). +It consists of roughly 10K records weighing about 1KB each. + +Compression Ratio | Compression Speed | Decompression Speed +------------------|-------------------|-------------------- +![Compression Ratio](doc/images/dict-cr.png "Compression Ratio") | ![Compression Speed](doc/images/dict-cs.png "Compression Speed") | ![Decompression Speed](doc/images/dict-ds.png "Decompression Speed") + + +These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds. + +Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_). +Hence, deploying one dictionary per type of data will provide the greatest benefits. +Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file. + +### Dictionary compression How To: + +1. Create the dictionary + + `zstd --train FullPathToTrainingSet/* -o dictionaryName` + +2. Compress with dictionary + + `zstd -D dictionaryName FILE` + +3. Decompress with dictionary + + `zstd -D dictionaryName --decompress FILE.zst` + + +## Build instructions + +`make` is the officially maintained build system of this project. +All other build systems are "compatible" and 3rd-party maintained, +they may feature small differences in advanced options. +When your system allows it, prefer using `make` to build `zstd` and `libzstd`. + +### Makefile + +If your system is compatible with standard `make` (or `gmake`), +invoking `make` in root directory will generate `zstd` cli in root directory. +It will also create `libzstd` into `lib/`. + +Other available options include: +- `make install` : create and install zstd cli, library and man pages +- `make check` : create and run `zstd`, test its behavior on local platform + +The `Makefile` follows the [GNU Standard Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html), +allowing staged install, standard flags, directory variables and command variables. + +For advanced use cases, specialized compilation flags which control binary generation +are documented in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library +and in [`programs/README.md`](programs/README.md#compilation-variables) for the `zstd` CLI. + +### cmake + +A `cmake` project generator is provided within `build/cmake`. +It can generate Makefiles or other build scripts +to create `zstd` binary, and `libzstd` dynamic and static libraries. + +By default, `CMAKE_BUILD_TYPE` is set to `Release`. + +#### Support for Fat (Universal2) Output + +`zstd` can be built and installed with support for both Apple Silicon (M1/M2) as well as Intel by using CMake's Universal2 support. +To perform a Fat/Universal2 build and install use the following commands: + +```bash +cmake -B build-cmake-debug -S build/cmake -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64" +cd build-cmake-debug +ninja +sudo ninja install +``` + +### Meson + +A Meson project is provided within [`build/meson`](build/meson). Follow +build instructions in that directory. + +You can also take a look at [`.travis.yml`](.travis.yml) file for an +example about how Meson is used to build this project. + +Note that default build type is **release**. + +### VCPKG +You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install zstd + +The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors. +If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + +### Visual Studio (Windows) + +Going into `build` directory, you will find additional possibilities: +- Projects for Visual Studio 2005, 2008 and 2010. + + VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017. +- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`, + which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. + +### Buck + +You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo. +The output binary will be in `buck-out/gen/programs/`. + +## Testing + +You can run quick local smoke tests by running `make check`. +If you can't use `make`, execute the `playTest.sh` script from the `src/tests` directory. +Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the `zstd` and `datagen` binary. +For information on CI testing, please refer to `TESTING.md`. + +## Status + +Zstandard is currently deployed within Facebook and many other large cloud infrastructures. +It is run continuously to compress large amounts of data in multiple formats and use cases. +Zstandard is considered safe for production environments. + +## License + +Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING). + +## Contributing + +The `dev` branch is the one where all contributions are merged before reaching `release`. +If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch. +Direct commit to `release` are not permitted. +For more information, please read [CONTRIBUTING](CONTRIBUTING.md). diff --git a/External/Zstd/zstd-1.5.5/lib/README.md b/External/Zstd/zstd-1.5.5/lib/README.md new file mode 100644 index 000000000..c3b5d1817 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/README.md @@ -0,0 +1,224 @@ +Zstandard library files +================================ + +The __lib__ directory is split into several sub-directories, +in order to make it easier to select or exclude features. + + +#### Building + +`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions), +including commands variables, staged install, directory variables and standard targets. +- `make` : generates both static and dynamic libraries +- `make install` : install libraries and headers in target system directories + +`libzstd` default scope is pretty large, including compression, decompression, dictionary builder, +and support for decoding legacy formats >= v0.5.0. +The scope can be reduced on demand (see paragraph _modular build_). + + +#### Multithreading support + +When building with `make`, by default the dynamic library is multithreaded and static library is single-threaded (for compatibility reasons). + +Enabling multithreading requires 2 conditions : +- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`) +- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`) + +For convenience, we provide a build target to generate multi and single threaded libraries: +- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`. +- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`. +- By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`. + +When linking a POSIX program with a multithreaded version of `libzstd`, +note that it's necessary to invoke the `-pthread` flag during link stage. + +Multithreading capabilities are exposed +via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351). + + +#### API + +Zstandard's stable API is exposed within [lib/zstd.h](zstd.h). + + +#### Advanced API + +Optional advanced features are exposed via : + +- `lib/zstd_errors.h` : translates `size_t` function results + into a `ZSTD_ErrorCode`, for accurate error handling. + +- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`, + it unlocks access to the experimental API, + exposed in the second part of `zstd.h`. + All definitions in the experimental APIs are unstable, + they may still change in the future, or even be removed. + As a consequence, experimental definitions shall ___never be used with dynamic library___ ! + Only static linking is allowed. + + +#### Modular build + +It's possible to compile only a limited set of features within `libzstd`. +The file structure is designed to make this selection manually achievable for any build system : + +- Directory `lib/common` is always required, for all variants. + +- Compression source code lies in `lib/compress` + +- Decompression source code lies in `lib/decompress` + +- It's possible to include only `compress` or only `decompress`, they don't depend on each other. + +- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples. + The API is exposed in `lib/dictBuilder/zdict.h`. + This module depends on both `lib/common` and `lib/compress` . + +- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`. + This module depends on `lib/common` and `lib/decompress`. + To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation. + Specifying a number limits versions supported to that version onward. + For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0". + Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats". + By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`. + Decoding supported legacy format is a transparent capability triggered within decompression functions. + It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`. + Each version does also provide its own set of advanced API. + For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` . + +- While invoking `make libzstd`, it's possible to define build macros + `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`, + and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the + corresponding features. This will also disable compilation of all + dependencies (e.g. `ZSTD_LIB_COMPRESSION=0` will also disable + dictBuilder). + +- There are a number of options that can help minimize the binary size of + `libzstd`. + + The first step is to select the components needed (using the above-described + `ZSTD_LIB_COMPRESSION` etc.). + + The next step is to set `ZSTD_LIB_MINIFY` to `1` when invoking `make`. This + disables various optional components and changes the compilation flags to + prioritize space-saving. + + Detailed options: Zstandard's code and build environment is set up by default + to optimize above all else for performance. In pursuit of this goal, Zstandard + makes significant trade-offs in code size. For example, Zstandard often has + more than one implementation of a particular component, with each + implementation optimized for different scenarios. For example, the Huffman + decoder has complementary implementations that decode the stream one symbol at + a time or two symbols at a time. Zstd normally includes both (and dispatches + between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1` or + `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding + compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` + and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of + only one or the other of two decompression implementations. The smallest + binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and + `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`). + + For squeezing the last ounce of size out, you can also define + `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`, + which removes the error messages that are otherwise returned by + `ZSTD_getErrorName` (implied by `ZSTD_LIB_MINIFY`). + + Finally, when integrating into your application, make sure you're doing link- + time optimization and unused symbol garbage collection (via some combination of, + e.g., `-flto`, `-ffat-lto-objects`, `-fuse-linker-plugin`, + `-ffunction-sections`, `-fdata-sections`, `-fmerge-all-constants`, + `-Wl,--gc-sections`, `-Wl,-z,norelro`, and an archiver that understands + the compiler's intermediate representation, e.g., `AR=gcc-ar`). Consult your + compiler's documentation. + +- While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1` + will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in + the shared library, which is now hidden by default. + +- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries + which can detect at runtime the presence of BMI2 instructions, and use them only if present. + These instructions contribute to better performance, notably on the decoder side. + By default, this feature is automatically enabled on detecting + the right instruction set (x64) and compiler (clang or gcc >= 5). + It's obviously disabled for different cpus, + or when BMI2 instruction set is _required_ by the compiler command line + (in this case, only the BMI2 code path is generated). + Setting this macro will either force to generate the BMI2 dispatcher (1) + or prevent it (0). It overrides automatic detection. + +- The build macro `ZSTD_NO_UNUSED_FUNCTIONS` can be defined to hide the definitions of functions + that zstd does not use. Not all unused functions are hidden, but they can be if needed. + Currently, this macro will hide function definitions in FSE and HUF that use an excessive + amount of stack space. + +- The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics. + Compiler builtins are still used. + +- The build macro `ZSTD_DECODER_INTERNAL_BUFFER` can be set to control + the amount of extra memory used during decompression to store literals. + This defaults to 64kB. Reducing this value reduces the memory footprint of + `ZSTD_DCtx` decompression contexts, + but might also result in a small decompression speed cost. + +- The C compiler macros `ZSTDLIB_VISIBLE`, `ZSTDERRORLIB_VISIBLE` and `ZDICTLIB_VISIBLE` + can be overridden to control the visibility of zstd's API. Additionally, + `ZSTDLIB_STATIC_API` and `ZDICTLIB_STATIC_API` can be overridden to control the visibility + of zstd's static API. Specifically, it can be set to `ZSTDLIB_HIDDEN` to hide the symbols + from the shared library. These macros default to `ZSTDLIB_VISIBILITY`, + `ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility + with the old macro names. + +#### Windows : using MinGW+MSYS to create DLL + +DLL can be created using MinGW+MSYS with the `make libzstd` command. +This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`. +The import library is only required with Visual C++. +The header file `zstd.h` and the dynamic library `dll\libzstd.dll` are required to +compile a project using gcc/MinGW. +The dynamic library has to be added to linking options. +It means that if a project that uses ZSTD consists of a single `test-dll.c` +file it should be linked with `dll\libzstd.dll`. For example: +``` + gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll +``` +The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. + + +#### Advanced Build options + +The build system requires a hash function in order to +separate object files created with different compilation flags. +By default, it tries to use `md5sum` or equivalent. +The hash function can be manually switched by setting the `HASH` variable. +For example : `make HASH=xxhsum` +The hash function needs to generate at least 64-bit using hexadecimal format. +When no hash function is found, +the Makefile just generates all object files into the same default directory, +irrespective of compilation flags. +This functionality only matters if `libzstd` is compiled multiple times +with different build flags. + +The build directory, where object files are stored +can also be manually controlled using variable `BUILD_DIR`, +for example `make BUILD_DIR=objectDir/v1`. +In which case, the hash function doesn't matter. + + +#### Deprecated API + +Obsolete API on their way out are stored in directory `lib/deprecated`. +At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`. +These prototypes will be removed in some future version. +Consider migrating code towards supported streaming API exposed in `zstd.h`. + + +#### Miscellaneous + +The other files are not source code. There are : + + - `BUCK` : support for `buck` build system (https://buckbuild.com/) + - `Makefile` : `make` script to build and install zstd library (static and dynamic) + - `README.md` : this file + - `dll/` : resources directory for Windows compilation + - `libzstd.pc.in` : script for `pkg-config` (used in `make install`) diff --git a/External/Zstd/zstd-1.5.5/lib/common/allocations.h b/External/Zstd/zstd-1.5.5/lib/common/allocations.h new file mode 100644 index 000000000..a3153c4ba --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/allocations.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides custom allocation primitives + */ + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + +#include "mem.h" /* MEM_STATIC */ +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" /* ZSTD_customMem */ + +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H + +/* custom memory allocation functions */ + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +#endif /* ZSTD_ALLOCATIONS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/bits.h b/External/Zstd/zstd-1.5.5/lib/common/bits.h new file mode 100644 index 000000000..def56c474 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/bits.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +#include "mem.h" + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +# else + return ZSTD_countTrailingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) { + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +# else + return ZSTD_countLeadingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/bitstream.h b/External/Zstd/zstd-1.5.5/lib/common/bitstream.h new file mode 100644 index 000000000..72b0b3df2 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/bitstream.h @@ -0,0 +1,437 @@ +/* ****************************************************************** + * bitstream + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "compiler.h" /* UNLIKELY() */ +#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ +#include "error_private.h" /* error codes and messages */ +#include "bits.h" /* ZSTD_highbit32 */ + + +/*========================================= +* Target specific +=========================================*/ +#ifndef ZSTD_NO_INTRINSICS +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ +# elif defined(__ICCARM__) +# include +# endif +#endif + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct { + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS) + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; + + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; + + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; + + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; + + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; + + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer)*8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X86) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); +#else + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +#endif +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not read beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +MEM_STATIC FORCE_INLINE_ATTR BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->limitPtr) { + return BIT_reloadDStreamFast(bitD); + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/compiler.h b/External/Zstd/zstd-1.5.5/lib/common/compiler.h new file mode 100644 index 000000000..73f8d0199 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/compiler.h @@ -0,0 +1,358 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +#include "portability_macros.h" + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ + +#if !defined(ZSTD_NO_INLINE) +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + +/** + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explicitly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#if defined(_MSC_VER) +# define WIN_CDECL __cdecl +#else +# define WIN_CDECL +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to eliminate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +#endif + +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + +/* force no inlining */ +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/* target attribute */ +#if defined(__GNUC__) || defined(__ICCARM__) +# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) +#else +# define TARGET_ATTRIBUTE(target) +#endif + +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. + */ +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if defined(NO_PREFETCH) +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# else +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ +} + +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif +#else +# define DONT_VECTORIZE +#endif + +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#if defined(__GNUC__) +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE { assert(0), __builtin_unreachable(); } +#else +# define ZSTD_UNREACHABLE { assert(0); } +#endif + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +# include /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ +#ifndef STATIC_BMI2 +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) +# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 +# define STATIC_BMI2 1 +# endif +# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__) +# define STATIC_BMI2 1 +# endif +#endif + +#ifndef STATIC_BMI2 + #define STATIC_BMI2 0 +#endif + +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include +# endif +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +/*-************************************************************** +* Alignment check +*****************************************************************/ + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ +#define ZSTD_DEPS_NEED_STDINT +#include "zstd_deps.h" /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/cpu.h b/External/Zstd/zstd-1.5.5/lib/common/cpu.h new file mode 100644 index 000000000..8bc34a36d --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/cpu.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include "mem.h" + +#ifdef _MSC_VER +#include +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/debug.c b/External/Zstd/zstd-1.5.5/lib/common/debug.c new file mode 100644 index 000000000..ebf7bfccf --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/debug.c @@ -0,0 +1,24 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * This module only hosts one global variable + * which can be used to dynamically influence the verbosity of traces, + * such as DEBUGLOG and RAWLOG + */ + +#include "debug.h" + +int g_debuglevel = DEBUGLEVEL; diff --git a/External/Zstd/zstd-1.5.5/lib/common/debug.h b/External/Zstd/zstd-1.5.5/lib/common/debug.h new file mode 100644 index 000000000..0e9817ea6 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/debug.h @@ -0,0 +1,107 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * The purpose of this header is to enable debug functions. + * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, + * and DEBUG_STATIC_ASSERT() for compile-time. + * + * By default, DEBUGLEVEL==0, which means run-time debug is disabled. + * + * Level 1 enables assert() only. + * Starting level 2, traces can be generated and pushed to stderr. + * The higher the level, the more verbose the traces. + * + * It's possible to dynamically adjust level using variable g_debug_level, + * which is only declared if DEBUGLEVEL>=2, + * and is a global variable, not multi-thread protected (use with care) + */ + +#ifndef DEBUG_H_12987983217 +#define DEBUG_H_12987983217 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + + +/* DEBUGLEVEL is expected to be defined externally, + * typically through compiler command line. + * Value must be a number. */ +#ifndef DEBUGLEVEL +# define DEBUGLEVEL 0 +#endif + + +/* recommended values for DEBUGLEVEL : + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display + * 2 : reserved, for currently active debug path + * 3 : events once per object lifetime (CCtx, CDict, etc.) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (verbose) + * 7+: events at every position (*very* verbose) + * + * It's generally inconvenient to output traces > 5. + * In which case, it's possible to selectively trigger high verbosity levels + * by modifying g_debug_level. + */ + +#if (DEBUGLEVEL>=1) +# define ZSTD_DEPS_NEED_ASSERT +# include "zstd_deps.h" +#else +# ifndef assert /* assert may be already defined, due to prior #include */ +# define assert(condition) ((void)0) /* disable assert (default) */ +# endif +#endif + +#if (DEBUGLEVEL>=2) +# define ZSTD_DEPS_NEED_IO +# include "zstd_deps.h" +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ + +# define RAWLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } } +# define DEBUGLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } } +#else +# define RAWLOG(l, ...) {} /* disabled */ +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* DEBUG_H_12987983217 */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/entropy_common.c b/External/Zstd/zstd-1.5.5/lib/common/entropy_common.c new file mode 100644 index 000000000..e2173afb0 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/entropy_common.c @@ -0,0 +1,340 @@ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#include "huf.h" +#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */ + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; + int previous0 = 0; + + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); + { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, + buffer, sizeof(buffer)); + if (FSE_isError(countSize)) return countSize; + if (countSize > hbSize) return ERROR(corruption_detected); + return countSize; + } } + assert(hbSize >= 8); + + /* init */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; + } else { + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; + } + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; + bitCount += 2; + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = ZSTD_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); + } + if (charnum >= maxSV1) break; + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } } + if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int flags) +{ +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)flags; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/External/Zstd/zstd-1.5.5/lib/common/error_private.c b/External/Zstd/zstd-1.5.5/lib/common/error_private.c new file mode 100644 index 000000000..075fc5ef4 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/error_private.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Data corruption detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; + case PREFIX(maxCode): + default: return notErrorCode; + } +#endif +} diff --git a/External/Zstd/zstd-1.5.5/lib/common/error_private.h b/External/Zstd/zstd-1.5.5/lib/common/error_private.h new file mode 100644 index 000000000..325daad40 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/error_private.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include "../zstd_errors.h" /* enum list */ +#include "compiler.h" +#include "debug.h" +#include "zstd_deps.h" /* size_t */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* already defined on Visual Studio */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + +/* check and forward error code */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0); + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0); + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/fse.h b/External/Zstd/zstd-1.5.5/lib/common/fse.h new file mode 100644 index 000000000..02a1f0bc5 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/fse.h @@ -0,0 +1,639 @@ +/* ****************************************************************** + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. + */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) +size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ + +typedef enum { + FSE_repeat_none, /**< Cannot use the previous table */ + FSE_repeat_check, /**< Can use the previous table but it must be checked */ + FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } FSE_repeat; + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + Hence their body are included in next section. +*/ +typedef struct { + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/**< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct { + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/**< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits+1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ + { U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; + } +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/common/fse_decompress.c b/External/Zstd/zstd-1.5.5/lib/common/fse_decompress.c new file mode 100644 index 000000000..1e1c9f92d --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/fse_decompress.c @@ -0,0 +1,311 @@ +/* ****************************************************************** + * FSE : Finite State Entropy decoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "debug.h" /* assert */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" +#include "bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; + FSE_DTable dtable[1]; /* Dynamically sized */ +} FSE_DecompressWksp; + + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + + DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* normal FSE decoding mode */ + { + size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } + + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = wksp->dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); +} + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/huf.h b/External/Zstd/zstd-1.5.5/lib/common/huf.h new file mode 100644 index 000000000..73d1ee565 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/huf.h @@ -0,0 +1,273 @@ +/* ****************************************************************** + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include "zstd_deps.h" /* size_t */ +#include "mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ + +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; + + +/* **************************************** + * HUF detailed API + * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } HUF_repeat; + +/** HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_buildCTable_wksp (HUF_CElt* tree, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int flags); + +/** HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); + +/** HUF_getNbBitsFromCTable() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : is not inlined, as HUF_CElt definition is private */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +/** HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#endif +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif + +#endif /* HUF_H_298734234 */ + +#if defined (__cplusplus) +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/common/mem.h b/External/Zstd/zstd-1.5.5/lib/common/mem.h new file mode 100644 index 000000000..98dd47a04 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/mem.h @@ -0,0 +1,435 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include "compiler.h" /* __has_builtin */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "zstd_deps.h" /* ZSTD_memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if defined(_AIX) +# include +# else +# include /* intptr_t */ +# endif + typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else +# include +#if CHAR_BIT != 8 +# error "this implementation requires char to be exactly 8-bit type" +#endif + typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; +#if USHRT_MAX != 65535 +# error "this implementation requires short to be exactly 16-bit type" +#endif + typedef unsigned short U16; + typedef signed short S16; +#if UINT_MAX != 4294967295 +# error "this implementation requires int to be exactly 32-bit type" +#endif + typedef unsigned int U32; + typedef signed int S32; +/* note : there are no limits defined for long long type in C90. + * limits exist in C99, however, in such case, is preferred */ + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * Default : method 1 if supported, else method 0 + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# ifdef __GNUC__ +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#else + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +#endif +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return MEM_swap32_fallback(in); +#endif +} + +MEM_STATIC U64 MEM_swap64_fallback(U64 in) +{ + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return MEM_swap64_fallback(in); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +/* code only tested on 32 and 64 bits systems */ +MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/pool.c b/External/Zstd/zstd-1.5.5/lib/common/pool.c new file mode 100644 index 000000000..d5ca5a780 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/pool.c @@ -0,0 +1,371 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Dependencies ======= */ +#include "../common/allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */ +#include "zstd_deps.h" /* size_t */ +#include "debug.h" /* assert */ +#include "pool.h" + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +#ifdef ZSTD_MULTITHREAD + +#include "threading.h" /* pthread adaptation */ + +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void *opaque; +} POOL_job; + +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t* threads; + size_t threadCapacity; + size_t threadLimit; + + /* The queue is a circular buffer */ + POOL_job *queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; + + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; + +/* POOL_thread() : + * Work thread for the thread pool. + * Waits for jobs and executes them. + * @returns : NULL on failure else non-null. + */ +static void* POOL_thread(void* opaque) { + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { return NULL; } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + + while ( ctx->queueEmpty + || (ctx->numThreadsBusy >= ctx->threadLimit) ) { + if (ctx->shutdown) { + /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), + * a few threads will be shutdown while !queueEmpty, + * but enough threads will remain active to finish the queue */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* Pop a job off the queue */ + { POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = (ctx->queueHead == ctx->queueTail); + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + + job.function(job.opaque); + + /* If the intended queue size was 0, signal after finishing job */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + } + } /* for (;;) */ + assert(0); /* Unreachable */ +} + +/* ZSTD_createThreadPool() : public access point */ +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem) +{ + POOL_ctx* ctx; + /* Check parameters */ + if (!numThreads) { return NULL; } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); + if (!ctx) { return NULL; } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate + * empty and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threadCapacity = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } + /* Initialize the threads */ + { size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = i; + POOL_free(ctx); + return NULL; + } } + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + } + return ctx; +} + +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) { + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { size_t i; + for (i = 0; i < ctx->threadCapacity; ++i) { + ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */ + } } +} + +void POOL_free(POOL_ctx *ctx) { + if (!ctx) { return; } + POOL_join(ctx); + ZSTD_pthread_mutex_destroy(&ctx->queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_customFree(ctx->queue, ctx->customMem); + ZSTD_customFree(ctx->threads, ctx->customMem); + ZSTD_customFree(ctx, ctx->customMem); +} + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +} + + +/* @return : 0 on success, 1 on error */ +static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +{ + if (numThreads <= ctx->threadCapacity) { + if (!numThreads) return 1; + ctx->threadLimit = numThreads; + return 0; + } + /* numThreads > threadCapacity */ + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + if (!threadPool) return 1; + /* replace existing thread pool */ + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); + ZSTD_customFree(ctx->threads, ctx->customMem); + ctx->threads = threadPool; + /* Initialize additional threads */ + { size_t threadId; + for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { + if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = threadId; + return 1; + } } + } } + /* successfully expanded */ + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + return 0; +} + +/* @return : 0 on success, 1 on error */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + int result; + if (ctx==NULL) return 1; + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + result = POOL_resize_internal(ctx, numThreads); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return result; +} + +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * When queueSize is 1 (pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free _and_ no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) { + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return (ctx->numThreadsBusy == ctx->threadLimit) || + !ctx->queueEmpty; + } +} + + +static void +POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) +{ + POOL_job job; + job.function = function; + job.opaque = opaque; + assert(ctx != NULL); + if (ctx->shutdown) return; + + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && (!ctx->shutdown)) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + if (isQueueFull(ctx)) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 0; + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 1; +} + + +#else /* ZSTD_MULTITHREAD not defined */ + +/* ========================== */ +/* No multi-threading support */ +/* ========================== */ + + +/* We don't need any data, but if it is empty, malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_poolCtx; + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* +POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_poolCtx; +} + +void POOL_free(POOL_ctx* ctx) { + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +int POOL_resize(POOL_ctx* ctx, size_t numThreads) { + (void)ctx; (void)numThreads; + return 0; +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); +} + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); + return 1; +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_poolCtx); + return sizeof(*ctx); +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/pool.h b/External/Zstd/zstd-1.5.5/lib/common/pool.h new file mode 100644 index 000000000..eb22ff509 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/pool.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef POOL_H +#define POOL_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +#include "zstd_deps.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ +#include "../zstd.h" + +typedef struct POOL_ctx_s POOL_ctx; + +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. +*/ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem); + +/*! POOL_free() : + * Free a thread pool returned by POOL_create(). + */ +void POOL_free(POOL_ctx* ctx); + + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx); + +/*! POOL_resize() : + * Expands or shrinks pool's number of threads. + * This is more efficient than releasing + creating a new context, + * since it tries to preserve and re-use existing threads. + * `numThreads` must be at least 1. + * @return : 0 when resize was successful, + * !0 (typically 1) if there is an error. + * note : only numThreads can be resized, queueSize remains unchanged. + */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads); + +/*! POOL_sizeof() : + * @return threadpool memory usage + * note : compatible with NULL (returns 0 in this case) + */ +size_t POOL_sizeof(const POOL_ctx* ctx); + +/*! POOL_function : + * The function type that can be added to a thread pool. + */ +typedef void (*POOL_function)(void*); + +/*! POOL_add() : + * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. + * Possibly blocks until there is room in the queue. + * Note : The function may be executed asynchronously, + * therefore, `opaque` must live until function has been completed. + */ +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); + + +/*! POOL_tryAdd() : + * Add the job `function(opaque)` to thread pool _if_ a queue slot is available. + * Returns immediately even if not (does not block). + * @return : 1 if successful, 0 if not. + */ +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); + + +#if defined (__cplusplus) +} +#endif + +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/common/portability_macros.h b/External/Zstd/zstd-1.5.5/lib/common/portability_macros.h new file mode 100644 index 000000000..8fd6ea82d --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/portability_macros.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X64)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/** + * Only enable assembly for GNUC compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/threading.c b/External/Zstd/zstd-1.5.5/lib/common/threading.c new file mode 100644 index 000000000..ca155b9b9 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/threading.c @@ -0,0 +1,176 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This file will hold wrapper for systems, which do not support pthreads + */ + +#include "threading.h" + +/* create fake symbol to avoid empty translation unit warning */ +int g_ZSTD_threading_useless_symbol; + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ + + +/* === Dependencies === */ +#include +#include + + +/* === Implementation === */ + +typedef struct { + void* (*start_routine)(void*); + void* arg; + int initialized; + ZSTD_pthread_cond_t initialized_cond; + ZSTD_pthread_mutex_t initialized_mutex; +} ZSTD_thread_params_t; + +static unsigned __stdcall worker(void *arg) +{ + void* (*start_routine)(void*); + void* thread_arg; + + /* Initialized thread_arg and start_routine and signal main thread that we don't need it + * to wait any longer. + */ + { + ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg; + thread_arg = thread_param->arg; + start_routine = thread_param->start_routine; + + /* Signal main thread that we are running and do not depend on its memory anymore */ + ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex); + thread_param->initialized = 1; + ZSTD_pthread_cond_signal(&thread_param->initialized_cond); + ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex); + } + + start_routine(thread_arg); + + return 0; +} + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg) +{ + ZSTD_thread_params_t thread_param; + (void)unused; + + thread_param.start_routine = start_routine; + thread_param.arg = arg; + thread_param.initialized = 0; + *thread = NULL; + + /* Setup thread initialization synchronization */ + if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) { + /* Should never happen on Windows */ + return -1; + } + if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) { + /* Should never happen on Windows */ + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return -1; + } + + /* Spawn thread */ + *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL); + if (!thread) { + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return errno; + } + + /* Wait for thread to be initialized */ + ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex); + while(!thread_param.initialized) { + ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex); + } + ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex); + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + + return 0; +} + +int ZSTD_pthread_join(ZSTD_pthread_t thread) +{ + DWORD result; + + if (!thread) return 0; + + result = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + + switch (result) { + case WAIT_OBJECT_0: + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif /* ZSTD_MULTITHREAD */ + +#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) +{ + *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); + if (!*mutex) + return 1; + return pthread_mutex_init(*mutex, attr); +} + +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) +{ + if (!*mutex) + return 0; + { + int const ret = pthread_mutex_destroy(*mutex); + ZSTD_free(*mutex); + return ret; + } +} + +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) +{ + *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); + if (!*cond) + return 1; + return pthread_cond_init(*cond, attr); +} + +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) +{ + if (!*cond) + return 0; + { + int const ret = pthread_cond_destroy(*cond); + ZSTD_free(*cond); + return ret; + } +} + +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/common/threading.h b/External/Zstd/zstd-1.5.5/lib/common/threading.h new file mode 100644 index 000000000..fb5c1c878 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/threading.h @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef THREADING_H_938743 +#define THREADING_H_938743 + +#include "debug.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ +#ifdef WINVER +# undef WINVER +#endif +#define WINVER 0x0600 + +#ifdef _WIN32_WINNT +# undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#include +#undef ERROR +#define ERROR(name) ZSTD_ERROR(name) + + +/* mutex */ +#define ZSTD_pthread_mutex_t CRITICAL_SECTION +#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0) +#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) +#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) + +/* condition variable */ +#define ZSTD_pthread_cond_t CONDITION_VARIABLE +#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) +#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) +#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) + +/* ZSTD_pthread_create() and ZSTD_pthread_join() */ +typedef HANDLE ZSTD_pthread_t; + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg); + +int ZSTD_pthread_join(ZSTD_pthread_t thread); + +/** + * add here more wrappers as required + */ + + +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +/* === POSIX Systems === */ +# include + +#if DEBUGLEVEL < 1 + +#define ZSTD_pthread_mutex_t pthread_mutex_t +#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) +#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) + +#define ZSTD_pthread_cond_t pthread_cond_t +#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) +#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#else /* DEBUGLEVEL >= 1 */ + +/* Debug implementation of threading. + * In this implementation we use pointers for mutexes and condition variables. + * This way, if we forget to init/destroy them the program will crash or ASAN + * will report leaks. + */ + +#define ZSTD_pthread_mutex_t pthread_mutex_t* +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr); +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex); +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a)) + +#define ZSTD_pthread_cond_t pthread_cond_t* +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr); +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#endif + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multithreading support */ + +typedef int ZSTD_pthread_mutex_t; +#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_mutex_destroy(a) ((void)(a)) +#define ZSTD_pthread_mutex_lock(a) ((void)(a)) +#define ZSTD_pthread_mutex_unlock(a) ((void)(a)) + +typedef int ZSTD_pthread_cond_t; +#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b)) +#define ZSTD_pthread_cond_signal(a) ((void)(a)) +#define ZSTD_pthread_cond_broadcast(a) ((void)(a)) + +/* do not use ZSTD_pthread_t */ + +#endif /* ZSTD_MULTITHREAD */ + +#if defined (__cplusplus) +} +#endif + +#endif /* THREADING_H_938743 */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/xxhash.c b/External/Zstd/zstd-1.5.5/lib/common/xxhash.c new file mode 100644 index 000000000..fd237c906 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/xxhash.c @@ -0,0 +1,24 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - xxHash homepage: https://cyan4973.github.io/xxHash/ + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/External/Zstd/zstd-1.5.5/lib/common/xxhash.h b/External/Zstd/zstd-1.5.5/lib/common/xxhash.h new file mode 100644 index 000000000..b8b73290b --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/xxhash.h @@ -0,0 +1,5686 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - xxHash homepage: https://cyan4973.github.io/xxHash/ + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif + +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +/* TODO: update */ +/* Notice extracted from xxHash homepage: + +xxHash is an extremely fast hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MurmurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +Note: SMHasher's CRC32 implementation is not the fastest one. +Other speed-oriented implementations can be faster, +especially in combination with PCLMUL instruction: +https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + + + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 1 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return `XXH_VERSION_NUMBER` of the invoked library. + */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# else +# if ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +# endif +#endif + +/*! + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that @ref xxh3_family provides competitive speed + * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif + +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* +Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute +introduced in CPP17 and C23. +CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough +C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough +*/ +#if XXH_HAS_C_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_CPP_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((fallthrough)) +#else +# define XXH_FALLTHROUGH +#endif + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. + * + * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/* XXH3_64bits(): + * default 64-bit variant, using default secret and default seed of 0. + * It's the fastest variant. */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); + +/* + * XXH3_64bits_withSeed(): + * This variant generates a custom secret on the fly + * based on default secret altered using the `seed` value. + * While this operation is decently fast, note that it's not completely free. + * Note: seed==0 produces the same results as XXH3_64bits(). + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing "XXH3_generateSecret()" instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); + +/* + * XXH3_64bits_reset(): + * Initialize with default parameters. + * digest will be equivalent to `XXH3_64bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); +/* + * XXH3_64bits_reset_withSeed(): + * Generate a custom secret from `seed`, and store it into `statePtr`. + * digest will be equivalent to `XXH3_64bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +/* + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * XXH128_cmp(): + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; } + + +/* XXH128() : + * simple alias to pre-selected XXH3_128bits variant + */ +XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/* + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, + * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @secretSize + * into an already allocated buffer @secretBuffer. + * @secretSize must be >= XXH3_SECRET_SIZE_MIN + * + * The generated secret can then be used with any `*_withSecret()` variant. + * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, + * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so + * XXH3_generateSecret() can be employed to ensure proper quality. + * + * customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes. + * The resulting `secret` will nonetheless provide all required qualities. + * + * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize); + + +/* + * XXH3_generateSecret_fromSeed(): + * + * Generate the same secret as the _withSeed() variants. + * + * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily). + * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * This generator is notably useful in combination with `_withSecretandSeed()`, + * as a way to emulate a faster `_withSeed()` variant. + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed); + +/* + * *_withSecretandSeed() : + * These variants generate hash values using either + * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes) + * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, + * via its impact to the seed. + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed); + +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + + +#endif /* XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64 & arm64, + * which are platforms known to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ +# if !defined(__clang__) && \ +( \ + (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + ( \ + defined(__GNUC__) && ( \ + (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ + ( \ + defined(__mips__) && \ + (__mips <= 5 || __mips_isa_rev < 6) && \ + (!defined(__mips16) || defined(__mips_mips16e2)) \ + ) \ + ) \ + ) \ +) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ + || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for ZSTD_malloc(), ZSTD_free() */ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */ +static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } +static void XXH_free (void* p) { ZSTD_free(p); } +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# define XXH_ASSERT(c) ((void)0) +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# include +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (e.g. eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; + return ((const xxh_unalign*)ptr)->u32; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 h32) +{ + h32 ^= h32 >> 15; + h32 *= XXH_PRIME32_2; + h32 ^= h32 >> 13; + h32 *= XXH_PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ +static xxh_u32 +XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + h32 += (*ptr++) * XXH_PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(h32); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 4: XXH_PROCESS4; + return XXH32_avalanche(h32); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 0: return XXH32_avalanche(h32); + } + XXH_ASSERT(0); + return h32; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +/*! + * @ingroup xxh32_family + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! + * @ingroup xxh32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + /* XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); */ + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer, but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; + return ((const xxh_unalign64*)ptr)->u64; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +static xxh_u64 XXH64_avalanche(xxh_u64 h64) +{ + h64 ^= h64 >> 33; + h64 *= XXH_PRIME64_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +static xxh_u64 +XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(h64); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (inputv[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH64_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + /* XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); */ + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#else +/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || defined(__aarch64__) || defined(_M_ARM) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + + +#if XXH_VECTOR == XXH_NEON +/* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split into + * two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] + * | '---------. .--------' | + * | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 is + * completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so modifying + * D11 will modify the high half of Q5. This is similar to how modifying AH + * will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd lanes + * in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead of + * taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the + * lower and upper halves of the Q register to end up with the high and low + * halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions modifying two + * registers at once is not possible in GCC or Clang's IR, and they have to + * create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, many + * quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes to a + * D-form register zero the high bits, similar to how writes to W-form scalar + * registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent + * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one + * operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be this + * mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } + * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } + * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single VZIP.32. + */ + +/*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) + * { + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ +# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__arm__) || defined(__thumb__) || defined(_M_ARM)) +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ + __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ + (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + } while (0) +# else +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + (outLo) = vmovn_u64 (in); \ + (outHi) = vshrn_n_u64 ((in), 32); \ + } while (0) +# endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * On AArch64 when not optimizing for size, XXH3 will run 6 lanes using NEON and + * 2 lanes on scalar by default. + * + * This can be set to 2, 4, 6, or 8. ARMv7 will default to all 8 NEON lanes, as the + * emulated 64-bit arithmetic is too slow. + * + * Modern ARM CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but it can't + * have more than 2 NEON (F0/F1) micro-ops. If you are only using NEON instructions, + * you are only using 2/3 of the CPU bandwidth. + * + * This is even more noticeable on the more advanced cores like the A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, @ref XXH3_NEON_LANES lanes will be processed using NEON, and the + * remaining lanes will use scalar instructions. This improves the bandwidth + * and also gives the integer pipelines something to do besides twiddling loop + * counters and pointers. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__OPTIMIZE_SIZE__) +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +# if defined(__s390x__) +# include +# else +/* gcc's altivec.h can have the unwanted consequence to unconditionally + * #define bool, vector, and pixel keywords, + * with bad consequences for programs already using these keywords for other purposes. + * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, + * but it seems that, in some cases, it isn't. + * Force the build macro to be defined, so that keywords are not altered. + */ +# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) +# define __APPLE_ALTIVEC__ +# endif +# include +# endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= 0x165667919E3779F9ULL; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= 0x9FB21C651E98DF25ULL; + h64 ^= (h64 >> 35) + len ; + h64 *= 0x9FB21C651E98DF25ULL; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); + + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + int const nbRounds = (int)len / 16; + int i; + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + acc = XXH3_avalanche(acc); + XXH_ASSERT(nbRounds >= 8); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + /* last bytes */ + acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + return XXH3_avalanche(acc); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', + * this will warn "discards 'const' qualifier". */ + union { + const __m512i* cp; + void* p; + } remote_const_void; + remote_const_void.cp = src + i; + dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { + uint64x2_t* const xacc = (uint64x2_t *) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* const xinput = (const uint8_t *) input; + uint8_t const* const xsecret = (const uint8_t *) secret; + + size_t i; + /* NEON for the first few lanes (these loops are normally interleaved) */ + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key; + uint32x2_t data_key_lo, data_key_hi; + /* xacc[i] += swap(data_vec); */ + uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); + uint64x2_t const swapped = vextq_u64(data64, data64, 1); + xacc[i] = vaddq_u64 (xacc[i], swapped); + /* data_key = data_vec ^ key_vec; */ + data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); + /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (data_key >> 32); + * data_key = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ + xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); + + } + /* Scalar for the remainder. This may be a zero iteration loop. */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { uint64x2_t* xacc = (uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1); + + size_t i; + /* NEON for the first few lanes (these loops are normally interleaved) */ + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); + uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec)); + + /* xacc[i] *= XXH_PRIME32_1 */ + uint32x2_t data_key_lo, data_key_hi; + /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (xacc[i] >> 32); + * xacc[i] = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + { /* + * prod_hi = (data_key >> 32) * XXH_PRIME32_1; + * + * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will + * incorrectly "optimize" this: + * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); + * shifted = vshll_n_u32(tmp, 32); + * to this: + * tmp = "vmulq_u64"(a, b); // no such thing! + * shifted = vshlq_n_u64(tmp, 32); + * + * However, unlike SSE, Clang lacks a 64-bit multiply routine + * for NEON, and it scalarizes two 64-bit multiplies instead. + * + * vmull_u32 has the same timing as vmul_u32, and it avoids + * this bug completely. + * See https://bugs.llvm.org/show_bug.cgi?id=39967 + */ + uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); + /* xacc[i] = prod_hi << 32; */ + xacc[i] = vshlq_n_u64(prod_hi, 32); + /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); + } + } + /* Scalar for the remainder. This may be a zero iteration loop. */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + } +} + +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + unsigned int* const xacc = (unsigned int*) acc; + xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ + xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = (xxh_u64x2)vec_xl(0, xacc + 4 * i); + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + /* xacc[i] = acc_vec; */ + vec_xst((xxh_u32x4)acc_vec, 0, xacc + 4 * i); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_u64x2* const xacc = (xxh_u64x2*) acc; + const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +/* scalar variants - universal */ + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__clang__) && defined(__aarch64__) + /* + * UGLY HACK: + * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes Clang to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + /* + * Note: in debug mode, this overrides the asm optimization + * and Clang will emit MOVK chains again. + */ + XXH_ASSERT(kSecretPtr == XXH3_kSecret); + + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes Clang to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + + + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * XXH3_accumulate() + * Loops over XXH3_accumulate_512(). + * Assumption: nbStripes will not overflow the secret size + */ +XXH_FORCE_INLINE void +XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes, + XXH3_f_accumulate_512 f_acc512) +{ + size_t n; + for (n = 0; n < nbStripes; n++ ) { + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; + XXH_PREFETCH(in + XXH_PREFETCH_DIST); + f_acc512(acc, + in, + secret + n*XXH_SECRET_CONSUME_RATE); + } +} + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* input, size_t len, + XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ + +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/* Note : when XXH3_consumeStripes() is invoked, + * there must be a guarantee that at least one more byte must be consumed from input + * so that the function can blindly consume all stripes using the "normal" secret segment */ +XXH_FORCE_INLINE void +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ + XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); + if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { + /* need a scrambling operation */ + size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; + size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); + f_scramble(acc, secret + secretLimit); + XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); + *nbStripesSoFarPtr = nbStripesAfterBlock; + } else { + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); + *nbStripesSoFarPtr += nbStripes; + } +} + +#ifndef XXH3_STREAM_USE_STACK +# ifndef __clang__ /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + + /* large input to consume : ingest per full block */ + if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar); + /* join to current block's end */ + { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar; + XXH_ASSERT(nbStripesToEnd <= nbStripes); + XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512); + f_scramble(acc, secret + state->secretLimit); + state->nbStripesSoFar = 0; + input += nbStripesToEnd * XXH_STRIPE_LEN; + nbStripes -= nbStripesToEnd; + } + /* consume per entire blocks */ + while(nbStripes >= state->nbStripesPerBlock) { + XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + state->secretLimit); + input += state->nbStripesPerBlock * XXH_STRIPE_LEN; + nbStripes -= state->nbStripesPerBlock; + } + /* consume last partial block */ + XXH3_accumulate(acc, input, secret, nbStripes, f_acc512); + input += nbStripes * XXH_STRIPE_LEN; + XXH_ASSERT(input < bEnd); /* at least some bytes left */ + state->nbStripesSoFar = nbStripes; + /* buffer predecessor of last partial stripe */ + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN); + } else { + /* content to consume <= block size */ + /* Consume input by a multiple of internal buffer size */ + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; + do { + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + input += XXH3_INTERNALBUFFER_SIZE; + } while (inputbuffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + } + } + + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate_512, XXH3_scrambleAcc); + /* last stripe */ + XXH3_accumulate_512(acc, + state->buffer + state->bufferedSize - XXH_STRIPE_LEN, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } else { /* bufferedSize < XXH_STRIPE_LEN */ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + XXH3_accumulate_512(acc, + lastStripe, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + int const nbRounds = (int)len / 32; + int i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + for (i=0; i<4; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + (32 * i), + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + XXH_ASSERT(nbRounds >= 4); + for (i=4 ; i < nbRounds; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + 0ULL - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ + +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n +#include +#include + +#if defined(__GNUC__) && __GNUC__ >= 4 +# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l)) +#else +# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) memset((p),(v),(l)) +#endif + +#endif /* ZSTD_DEPS_COMMON */ + +/* Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#include + +#define ZSTD_malloc(s) malloc(s) +#define ZSTD_calloc(n,s) calloc((n), (s)) +#define ZSTD_free(p) free((p)) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor)) + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include +#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* Only requested when is known to be present. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +#include + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/zstd_internal.h b/External/Zstd/zstd-1.5.5/lib/common/zstd_internal.h new file mode 100644 index 000000000..1f942f27b --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/zstd_internal.h @@ -0,0 +1,392 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +#include "compiler.h" +#include "cpu.h" +#include "mem.h" +#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ +#ifndef ZSTD_NO_TRACE +# include "zstd_trace.h" +#else +# define ZSTD_TRACE 0 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 + +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1<= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ +typedef struct seqDef_s { + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ + U16 litLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ +} seqDef; + +/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} seqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/** + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->mlBase + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0x10000; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0x10000; + } + } + return seqLen; +} + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/zstd_trace.h b/External/Zstd/zstd-1.5.5/lib/common/zstd_trace.h new file mode 100644 index 000000000..da20534eb --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/zstd_trace.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_TRACE_H +#define ZSTD_TRACE_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include + +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386 and aarch64 + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 +#else +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR +#endif + +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif + +#if ZSTD_TRACE + +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; + +typedef struct { + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + unsigned streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + unsigned dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; + +/** + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). + * + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. + */ +typedef unsigned long long ZSTD_TraceCtx; + +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); + +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); + +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +#endif /* ZSTD_TRACE */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_TRACE_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/clevels.h b/External/Zstd/zstd-1.5.5/lib/compress/clevels.h new file mode 100644 index 000000000..c18da465f --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/clevels.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CLEVELS_H +#define ZSTD_CLEVELS_H + +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "../zstd.h" + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_MAX_CLEVEL 22 + +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" - for any srcSize > 256 KB */ + /* W, C, H, S, L, TL, strat */ + { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ + { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ + { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ + { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */ + { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */ + { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ + { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ + { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ + { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ + { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ + { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ + { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ + { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ + { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ + { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ + { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ + { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ + { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +}; + + + +#endif /* ZSTD_CLEVELS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/fse_compress.c b/External/Zstd/zstd-1.5.5/lib/compress/fse_compress.c new file mode 100644 index 000000000..5d3770808 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/fse_compress.c @@ -0,0 +1,624 @@ +/* ****************************************************************** + * FSE : Finite State Entropy encoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/compiler.h" +#include "../common/mem.h" /* U32, U16, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "hist.h" /* HIST_count_wksp */ +#include "../common/bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +#include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */ +#include "../common/bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 const maxSV1 = maxSymbolValue+1; + + U16* cumul = (U16*)workSpace; /* size = maxSV1 */ + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */ + + U32 highThreshold = tableSize-1; + + assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */ + if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); + /* CTable header */ + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + assert(tableLog < 16); /* required for threshold strategy to work */ + + /* For explanations on how to distribute symbol values over the table : + * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + #ifdef __clang_analyzer__ + ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + #endif + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u <= maxSV1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + assert(normalizedCounter[u-1] >= 0); + cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1]; + assert(cumul[u] >= cumul[u-1]); /* no overflow */ + } } + cumul[maxSV1] = (U16)(tableSize+1); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + /* Case for no low prob count symbols. Lay down 8 bytes at a time + * to reduce branch misses since we are operating on a small block + */ + BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */ + { U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Spread symbols across the table. Lack of lowprob symbols means that + * we don't need variable sized inner loop, so we can unroll the loop and + * reduce branch misses. + */ + { size_t position = 0; + size_t s; + size_t const unroll = 2; /* Experimentally determined optimal unroll */ + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableSymbol[uPosition] = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); /* Must have initialized all positions */ + } + } else { + U32 position = 0; + U32 symbol; + for (symbol=0; symbol highThreshold) + position = (position + step) & tableMask; /* Low proba area */ + } } + assert(position==0); /* Must have initialized all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u 1); + { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1); + U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut; + symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; + symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]); + total += (unsigned)normalizedCounter[s]; + } } } } + +#if 0 /* debug : symbol costs */ + DEBUGLOG(5, "\n --- table statistics : "); + { U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", + symbol, normalizedCounter[symbol], + FSE_getMaxNbBits(symbolTT, symbol), + (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); + } } +#endif + + return 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-************************************************************** +* FSE NCount encoding +****************************************************************/ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog + + 4 /* bitCount initialized at 4 */ + + 2 /* first two symbols may use one additional bit each */) / 8) + + 1 /* round up to whole nb bytes */ + + 2 /* additional two bytes for bitstream flush */; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static size_t +FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; + + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + if (symbol == alphabetSize) break; /* incorrect distribution */ + while (symbol >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (symbol >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (symbol-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { int count = normalizedCounter[symbol++]; + int const max = (2*threshold-1) - remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count>=threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; } + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); +} + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) +{ + short const NOT_YET_ASSIGNED = -2; + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = lowProbCount; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + + norm[s]=NOT_YET_ASSIGNED; + } + ToDistribute = (1 << tableLog) - distributed; + + if (ToDistribute == 0) + return 0; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) { maxV=s; maxC=count[s]; } + norm[maxV] += (short)ToDistribute; + return 0; + } + + if (total == 0) { + /* all of the symbols were low enough for the lowOne or lowThreshold */ + for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) + if (norm[s] > 0) { ToDistribute--; norm[s]++; } + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ZSTD_div64((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue, unsigned useLowProbCount) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + short const lowProbCount = useLowProbCount ? -1 : 1; + U64 const scale = 62 - tableLog; + U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = lowProbCount; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) { largestP=proba; largest=s; } + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U< FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/hist.c b/External/Zstd/zstd-1.5.5/lib/compress/hist.c new file mode 100644 index 000000000..e2fb431f0 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/hist.c @@ -0,0 +1,181 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/mem.h" /* U32, BYTE, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "../common/error_private.h" /* ERROR */ +#include "hist.h" + + +/* --- Error management --- */ +unsigned HIST_isError(size_t code) { return ERR_isError(code); } + +/*-************************************************************** + * Histogram functions + ****************************************************************/ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned largestCount=0; + + ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip largestCount) largestCount = count[s]; + } + + return largestCount; +} + +typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; + +/* HIST_count_parallel_wksp() : + * store histogram into 4 intermediate tables, recombined at the end. + * this design makes better use of OoO cpus, + * and is noticeably faster when some values are heavily repeated. + * But it needs some additional workspace for intermediate tables. + * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. + * @return : largest histogram frequency, + * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ +static size_t HIST_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + HIST_checkInput_e check, + U32* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + /* safety checks */ + assert(*maxSymbolValuePtr <= 255); + if (!sourceSize) { + ZSTD_memset(count, 0, countSize); + *maxSymbolValuePtr = 0; + return 0; + } + ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ip max) max = Counting1[s]; + } } + + { unsigned maxSymbolValue = 255; + while (!Counting1[maxSymbolValue]) maxSymbolValue--; + if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); + *maxSymbolValuePtr = maxSymbolValue; + ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ + } + return (size_t)max; +} + +/* HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if (sourceSize < 1500) /* heuristic threshold */ + return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); +} + +/* HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + if (*maxSymbolValuePtr < 255) + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); + *maxSymbolValuePtr = 255; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); +} + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); +} + +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/compress/hist.h b/External/Zstd/zstd-1.5.5/lib/compress/hist.h new file mode 100644 index 000000000..887896b81 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/hist.h @@ -0,0 +1,75 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/zstd_deps.h" /* size_t */ + + +/* --- simple histogram functions --- */ + +/*! HIST_count(): + * Provides the precise count of each byte within a table 'count'. + * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + * Updates *maxSymbolValuePtr with actual largest symbol value detected. + * @return : count of the most frequent symbol (which isn't identified). + * or an error code, which can be tested using HIST_isError(). + * note : if return == srcSize, there is only one symbol. + */ +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */ + + +/* --- advanced histogram functions --- */ + +#define HIST_WKSP_SIZE_U32 1024 +#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) +/** HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * Benefit is this function will use very little stack space. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/** HIST_countFast() : + * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` + */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +/** HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/*! HIST_count_simple() : + * Same as HIST_countFast(), this function is unsafe, + * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. + * It is also a bit slower for large inputs. + * However, it does not need any additional memory (not even on stack). + * @return : count of the most frequent symbol. + * Note this function doesn't produce any error (i.e. it must succeed). + */ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); diff --git a/External/Zstd/zstd-1.5.5/lib/compress/huf_compress.c b/External/Zstd/zstd-1.5.5/lib/compress/huf_compress.c new file mode 100644 index 000000000..29871877a --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/huf_compress.c @@ -0,0 +1,1435 @@ +/* ****************************************************************** + * Huffman encoder, part of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" +#include "hist.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "../common/fse.h" /* header compression */ +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Required declarations +****************************************************************/ +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + + +/* ************************************************************** +* Debug Traces +****************************************************************/ + +#if DEBUGLEVEL >= 2 + +static size_t showU32(const U32* arr, size_t size) +{ + size_t u; + for (u=0; u= add) { + assert(add < align); + assert(((size_t)aligned & mask) == 0); + *workspaceSizePtr -= add; + return aligned; + } else { + *workspaceSizePtr = 0; + return NULL; + } +} + + +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 + +typedef struct { + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + unsigned count[HUF_TABLELOG_MAX+1]; + S16 norm[HUF_TABLELOG_MAX+1]; +} HUF_CompressWeightsWksp; + +static size_t +HUF_compressWeights(void* dst, size_t dstSize, + const void* weightTable, size_t wtSize, + void* workspace, size_t workspaceSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + unsigned maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + + if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC); + + /* init conditions */ + if (wtSize <= 1) return 0; /* Not compressible */ + + /* Scan input and build symbol stats */ + { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */ + if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ + if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); + + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) ); + op += hSize; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + return (size_t)(op-ostart); +} + +static size_t HUF_getNbBits(HUF_CElt elt) +{ + return elt & 0xFF; +} + +static size_t HUF_getNbBitsFast(HUF_CElt elt) +{ + return elt; +} + +static size_t HUF_getValue(HUF_CElt elt) +{ + return elt & ~(size_t)0xFF; +} + +static size_t HUF_getValueFast(HUF_CElt elt) +{ + return elt; +} + +static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits) +{ + assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX); + *elt = nbBits; +} + +static void HUF_setValue(HUF_CElt* elt, size_t value) +{ + size_t const nbBits = HUF_getNbBits(*elt); + if (nbBits > 0) { + assert((value >> nbBits) == 0); + *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits); + } +} + +typedef struct { + HUF_CompressWeightsWksp wksp; + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; +} HUF_WriteCTableWksp; + +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, + void* workspace, size_t workspaceSize) +{ + HUF_CElt const* const ct = CTable + 1; + BYTE* op = (BYTE*)dst; + U32 n; + HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp)); + + /* check conditions */ + if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + wksp->bitsToWeight[0] = 0; + for (n=1; nbitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; nhuffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])]; + + /* attempt weights compression by FSE */ + if (maxDstSize < 1) return ERROR(dstSize_tooSmall); + { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) ); + if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; + } } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; nhuffWeight[n] << 4) + wksp->huffWeight[n+1]); + return ((maxSymbolValue+1)/2) + 1; +} + + +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + HUF_CElt* const ct = CTable + 1; + + /* get symbol weights */ + CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + *hasZeroWeights = (rankVal[0] > 0); + + /* check result */ + if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); + + CTable[0] = tableLog; + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 curr = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = curr; + } } + + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n @targetNbBits + * to employ @targetNbBits instead. Then it adjusts the tree + * so that it remains a valid canonical Huffman tree. + * + * @pre The sum of the ranks of each symbol == 2^largestBits, + * where largestBits == huffNode[lastNonNull].nbBits. + * @post The sum of the ranks of each symbol == 2^largestBits, + * where largestBits is the return value (expected <= targetNbBits). + * + * @param huffNode The Huffman tree modified in place to enforce targetNbBits. + * It's presumed sorted, from most frequent to rarest symbol. + * @param lastNonNull The symbol with the lowest count in the Huffman tree. + * @param targetNbBits The allowed number of bits, which the Huffman tree + * may not respect. After this function the Huffman tree will + * respect targetNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment. + */ +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + /* early exit : no elt > targetNbBits, so the tree is already valid. */ + if (largestBits <= targetNbBits) return largestBits; + + DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits); + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - targetNbBits); + int n = (int)lastNonNull; + + /* Adjust any ranks > targetNbBits to targetNbBits. + * Compute totalCost, which is how far the sum of the ranks is + * we are over 2^largestBits after adjust the offending ranks. + */ + while (huffNode[n].nbBits > targetNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)targetNbBits; + n--; + } + /* n stops at huffNode[n].nbBits <= targetNbBits */ + assert(huffNode[n].nbBits <= targetNbBits); + /* n end at index of smallest symbol using < targetNbBits */ + while (huffNode[n].nbBits == targetNbBits) --n; + + /* renorm totalCost from 2^largestBits to 2^targetNbBits + * note : totalCost is necessarily a multiple of baseCost */ + assert(((U32)totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - targetNbBits); + assert(totalCost > 0); + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + + /* Get pos of last (smallest = lowest cum. count) symbol per rank */ + ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = targetNbBits; + int pos; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */ + rankLast[targetNbBits-currentNbBits] = (U32)pos; + } } + + while (totalCost > 0) { + /* Try to reduce the next power of 2 above totalCost because we + * gain back half the rank. + */ + U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 const highPos = rankLast[nBitsToDecrease]; + U32 const lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + /* Decrease highPos if no symbols of lowPos or if it is + * not cheaper to remove 2 lowPos than highPos. + */ + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) + nBitsToDecrease++; + assert(rankLast[nBitsToDecrease] != noSymbol); + /* Increase the number of bits to gain back half the rank cost. */ + totalCost -= 1 << (nBitsToDecrease-1); + huffNode[rankLast[nBitsToDecrease]].nbBits++; + + /* Fix up the new rank. + * If the new rank was empty, this symbol is now its smallest. + * Otherwise, this symbol will be the largest in the new rank so no adjustment. + */ + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; + /* Fix up the old rank. + * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, + * it must be the only symbol in its rank, so the old rank now has no symbols. + * Otherwise, since the Huffman nodes are sorted by count, the previous position is now + * the smallest node in the rank. If the previous position belongs to a different rank, + * then the rank is now empty. + */ + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } + } /* while (totalCost > 0) */ + + /* If we've removed too much weight, then we have to add it back. + * To avoid overshooting again, we only adjust the smallest rank. + * We take the largest nodes from the lowest rank 0 and move them + * to rank 1. There's guaranteed to be enough rank 0 symbols because + * TODO. + */ + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + /* special case : no rank 1 symbol (using targetNbBits-1); + * let's create one from largest rank 0 (using targetNbBits). + */ + if (rankLast[1] == noSymbol) { + while (huffNode[n].nbBits == targetNbBits) n--; + huffNode[n+1].nbBits--; + assert(n >= 0); + rankLast[1] = (U32)(n+1); + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } + } /* repay normalized cost */ + } /* there are several too large elements (at least >= 2) */ + + return targetNbBits; +} + +typedef struct { + U16 base; + U16 curr; +} rankPos; + +typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)]; + +/* Number of buckets available for HUF_sort() */ +#define RANK_POSITION_TABLE_SIZE 192 + +typedef struct { + huffNodeTable huffNodeTbl; + rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; +} HUF_buildCTable_wksp_tables; + +/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing. + * Strategy is to use as many buckets as possible for representing distinct + * counts while using the remainder to represent all "large" counts. + * + * To satisfy this requirement for 192 buckets, we can do the following: + * Let buckets 0-166 represent distinct counts of [0, 166] + * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. + */ +#define RANK_POSITION_MAX_COUNT_LOG 32 +#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */) +#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */) + +/* Return the appropriate bucket index for a given count. See definition of + * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy. + */ +static U32 HUF_getIndex(U32 const count) { + return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF) + ? count + : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; +} + +/* Helper swap function for HUF_quickSortPartition() */ +static void HUF_swapNodes(nodeElt* a, nodeElt* b) { + nodeElt tmp = *a; + *a = *b; + *b = tmp; +} + +/* Returns 0 if the huffNode array is not sorted by descending count */ +MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) { + U32 i; + for (i = 1; i < maxSymbolValue1; ++i) { + if (huffNode[i].count > huffNode[i-1].count) { + return 0; + } + } + return 1; +} + +/* Insertion sort by descending order */ +HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) { + int i; + int const size = high-low+1; + huffNode += low; + for (i = 1; i < size; ++i) { + nodeElt const key = huffNode[i]; + int j = i - 1; + while (j >= 0 && huffNode[j].count < key.count) { + huffNode[j + 1] = huffNode[j]; + j--; + } + huffNode[j + 1] = key; + } +} + +/* Pivot helper function for quicksort. */ +static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) { + /* Simply select rightmost element as pivot. "Better" selectors like + * median-of-three don't experimentally appear to have any benefit. + */ + U32 const pivot = arr[high].count; + int i = low - 1; + int j = low; + for ( ; j < high; j++) { + if (arr[j].count > pivot) { + i++; + HUF_swapNodes(&arr[i], &arr[j]); + } + } + HUF_swapNodes(&arr[i + 1], &arr[high]); + return i + 1; +} + +/* Classic quicksort by descending with partially iterative calls + * to reduce worst case callstack size. + */ +static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) { + int const kInsertionSortThreshold = 8; + if (high - low < kInsertionSortThreshold) { + HUF_insertionSort(arr, low, high); + return; + } + while (low < high) { + int const idx = HUF_quickSortPartition(arr, low, high); + if (idx - low < high - idx) { + HUF_simpleQuickSort(arr, low, idx - 1); + low = idx + 1; + } else { + HUF_simpleQuickSort(arr, idx + 1, high); + high = idx - 1; + } + } +} + +/** + * HUF_sort(): + * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket. + * + * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. + * Must have (maxSymbolValue + 1) entries. + * @param[in] count Histogram of the symbols. + * @param[in] maxSymbolValue Maximum symbol value. + * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. + */ +static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSymbolValue, rankPos rankPosition[]) { + U32 n; + U32 const maxSymbolValue1 = maxSymbolValue+1; + + /* Compute base and set curr to base. + * For symbol s let lowerRank = HUF_getIndex(count[n]) and rank = lowerRank + 1. + * See HUF_getIndex to see bucketing strategy. + * We attribute each symbol to lowerRank's base value, because we want to know where + * each rank begins in the output, so for rank R we want to count ranks R+1 and above. + */ + ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); + for (n = 0; n < maxSymbolValue1; ++n) { + U32 lowerRank = HUF_getIndex(count[n]); + assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1); + rankPosition[lowerRank].base++; + } + + assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); + /* Set up the rankPosition table */ + for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { + rankPosition[n-1].base += rankPosition[n].base; + rankPosition[n-1].curr = rankPosition[n-1].base; + } + + /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */ + for (n = 0; n < maxSymbolValue1; ++n) { + U32 const c = count[n]; + U32 const r = HUF_getIndex(c) + 1; + U32 const pos = rankPosition[r].curr++; + assert(pos < maxSymbolValue1); + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } + + /* Sort each bucket. */ + for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) { + int const bucketSize = rankPosition[n].curr - rankPosition[n].base; + U32 const bucketStartIdx = rankPosition[n].base; + if (bucketSize > 1) { + assert(bucketStartIdx < maxSymbolValue1); + HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1); + } + } + + assert(HUF_isSorted(huffNode, maxSymbolValue1)); +} + + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) + +/* HUF_buildTree(): + * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. + * + * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. + * @param maxSymbolValue The maximum symbol value. + * @return The smallest node in the Huffman tree (by count). + */ +static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) +{ + nodeElt* const huffNode0 = huffNode - 1; + int nonNullRank; + int lowS, lowN; + int nodeNb = STARTNODE; + int n, nodeRoot; + DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1); + /* init for parents */ + nonNullRank = (int)maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1)); + + return nonNullRank; +} + +/** + * HUF_buildCTableFromTree(): + * Build the CTable given the Huffman tree in huffNode. + * + * @param[out] CTable The output Huffman CTable. + * @param huffNode The Huffman tree. + * @param nonNullRank The last and smallest node in the Huffman tree. + * @param maxSymbolValue The maximum symbol value. + * @param maxNbBits The exact maximum number of bits used in the Huffman tree. + */ +static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) +{ + HUF_CElt* const ct = CTable + 1; + /* fill result into ctable (val, nbBits) */ + int n; + U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + int const alphabetSize = (int)(maxSymbolValue + 1); + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine starting value per rank */ + { U16 min = 0; + for (n=(int)maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; nhuffNodeTbl; + nodeElt* const huffNode = huffNode0+1; + int nonNullRank; + + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables)); + + DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1); + + /* safety checks */ + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) + return ERROR(workSpace_tooSmall); + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1)); + + /* build tree */ + nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); + + /* determine and enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + + HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits); + + return maxNbBits; +} + +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +{ + HUF_CElt const* ct = CTable + 1; + size_t nbBits = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + nbBits += HUF_getNbBits(ct[s]) * count[s]; + } + return nbBits >> 3; +} + +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { + HUF_CElt const* ct = CTable + 1; + int bad = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0); + } + return !bad; +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +/** HUF_CStream_t: + * Huffman uses its own BIT_CStream_t implementation. + * There are three major differences from BIT_CStream_t: + * 1. HUF_addBits() takes a HUF_CElt (size_t) which is + * the pair (nbBits, value) in the format: + * format: + * - Bits [0, 4) = nbBits + * - Bits [4, 64 - nbBits) = 0 + * - Bits [64 - nbBits, 64) = value + * 2. The bitContainer is built from the upper bits and + * right shifted. E.g. to add a new value of N bits + * you right shift the bitContainer by N, then or in + * the new value into the N upper bits. + * 3. The bitstream has two bit containers. You can add + * bits to the second container and merge them into + * the first container. + */ + +#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8) + +typedef struct { + size_t bitContainer[2]; + size_t bitPos[2]; + + BYTE* startPtr; + BYTE* ptr; + BYTE* endPtr; +} HUF_CStream_t; + +/**! HUF_initCStream(): + * Initializes the bitstream. + * @returns 0 or an error code. + */ +static size_t HUF_initCStream(HUF_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + ZSTD_memset(bitC, 0, sizeof(*bitC)); + bitC->startPtr = (BYTE*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]); + if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! HUF_addBits(): + * Adds the symbol stored in HUF_CElt elt to the bitstream. + * + * @param elt The element we're adding. This is a (nbBits, value) pair. + * See the HUF_CStream_t docs for the format. + * @param idx Insert into the bitstream at this idx. + * @param kFast This is a template parameter. If the bitstream is guaranteed + * to have at least 4 unused bits after this call it may be 1, + * otherwise it must be 0. HUF_addBits() is faster when fast is set. + */ +FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int idx, int kFast) +{ + assert(idx <= 1); + assert(HUF_getNbBits(elt) <= HUF_TABLELOG_ABSOLUTEMAX); + /* This is efficient on x86-64 with BMI2 because shrx + * only reads the low 6 bits of the register. The compiler + * knows this and elides the mask. When fast is set, + * every operation can use the same value loaded from elt. + */ + bitC->bitContainer[idx] >>= HUF_getNbBits(elt); + bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt); + /* We only read the low 8 bits of bitC->bitPos[idx] so it + * doesn't matter that the high bits have noise from the value. + */ + bitC->bitPos[idx] += HUF_getNbBitsFast(elt); + assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + /* The last 4-bits of elt are dirty if fast is set, + * so we must not be overwriting bits that have already been + * inserted into the bit container. + */ +#if DEBUGLEVEL >= 1 + { + size_t const nbBits = HUF_getNbBits(elt); + size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1; + (void)dirtyBits; + /* Middle bits are 0. */ + assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); + /* We didn't overwrite any bits in the bit container. */ + assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + (void)dirtyBits; + } +#endif +} + +FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC) +{ + bitC->bitContainer[1] = 0; + bitC->bitPos[1] = 0; +} + +/*! HUF_mergeIndex1() : + * Merges the bit container @ index 1 into the bit container @ index 0 + * and zeros the bit container @ index 1. + */ +FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC) +{ + assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER); + bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF); + bitC->bitContainer[0] |= bitC->bitContainer[1]; + bitC->bitPos[0] += bitC->bitPos[1]; + assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER); +} + +/*! HUF_flushBits() : +* Flushes the bits in the bit container @ index 0. +* +* @post bitPos will be < 8. +* @param kFast If kFast is set then we must know a-priori that +* the bit container will not overflow. +*/ +FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast) +{ + /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */ + size_t const nbBits = bitC->bitPos[0] & 0xFF; + size_t const nbBytes = nbBits >> 3; + /* The top nbBits bits of bitContainer are the ones we need. */ + size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits); + /* Mask bitPos to account for the bytes we consumed. */ + bitC->bitPos[0] &= 7; + assert(nbBits > 0); + assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitContainer); + bitC->ptr += nbBytes; + assert(!kFast || bitC->ptr <= bitC->endPtr); + if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + /* bitContainer doesn't need to be modified because the leftover + * bits are already the top bitPos bits. And we don't care about + * noise in the lower values. + */ +} + +/*! HUF_endMark() + * @returns The Huffman stream end mark: A 1-bit value = 1. + */ +static HUF_CElt HUF_endMark(void) +{ + HUF_CElt endMark; + HUF_setNbBits(&endMark, 1); + HUF_setValue(&endMark, 1); + return endMark; +} + +/*! HUF_closeCStream() : + * @return Size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +static size_t HUF_closeCStream(HUF_CStream_t* bitC) +{ + HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0); + HUF_flushBits(bitC, /* kFast */ 0); + { + size_t const nbBits = bitC->bitPos[0] & 0xFF; + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0); + } +} + +FORCE_INLINE_TEMPLATE void +HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast) +{ + HUF_addBits(bitCPtr, CTable[symbol], idx, fast); +} + +FORCE_INLINE_TEMPLATE void +HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC, + const BYTE* ip, size_t srcSize, + const HUF_CElt* ct, + int kUnroll, int kFastFlush, int kLastFast) +{ + /* Join to kUnroll */ + int n = (int)srcSize; + int rem = n % kUnroll; + if (rem > 0) { + for (; rem > 0; --rem) { + HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0); + } + HUF_flushBits(bitC, kFastFlush); + } + assert(n % kUnroll == 0); + + /* Join to 2 * kUnroll */ + if (n % (2 * kUnroll)) { + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast); + HUF_flushBits(bitC, kFastFlush); + n -= kUnroll; + } + assert(n % (2 * kUnroll) == 0); + + for (; n>0; n-= 2 * kUnroll) { + /* Encode kUnroll symbols into the bitstream @ index 0. */ + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast); + HUF_flushBits(bitC, kFastFlush); + /* Encode kUnroll symbols into the bitstream @ index 1. + * This allows us to start filling the bit container + * without any data dependencies. + */ + HUF_zeroIndex1(bitC); + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast); + /* Merge bitstream @ index 1 into the bitstream @ index 0 */ + HUF_mergeIndex1(bitC); + HUF_flushBits(bitC, kFastFlush); + } + assert(n == 0); + +} + +/** + * Returns a tight upper bound on the output space needed by Huffman + * with 8 bytes buffer to handle over-writes. If the output is at least + * this large we don't need to do bounds checks during Huffman encoding. + */ +static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog) +{ + return ((srcSize * tableLog) >> 3) + 8; +} + + +FORCE_INLINE_TEMPLATE size_t +HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + U32 const tableLog = (U32)CTable[0]; + HUF_CElt const* ct = CTable + 1; + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + HUF_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op)); + if (HUF_isError(initErr)) return 0; } + + if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11) + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0); + else { + if (MEM_32bits()) { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: ZSTD_FALLTHROUGH; + case 9: ZSTD_FALLTHROUGH; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 7: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } else { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 9: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 7: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 6: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } + } + assert(bitC.ptr <= bitC.endPtr); + + return HUF_closeCStream(&bitC); +} + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int flags) +{ + if (flags & HUF_flags_bmi2) { + return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); + } + return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); +} + +#else + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int flags) +{ + (void)flags; + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +#endif + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) +{ + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); +} + +static size_t +HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, int flags) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + assert(ip <= iend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + op += cSize; + } + + return (size_t)(op-ostart); +} + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) +{ + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); +} + +typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; + +static size_t HUF_compressCTable_internal( + BYTE* const ostart, BYTE* op, BYTE* const oend, + const void* src, size_t srcSize, + HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags) +{ + size_t const cSize = (nbStreams==HUF_singleStream) ? + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags); + if (HUF_isError(cSize)) { return cSize; } + if (cSize==0) { return 0; } /* uncompressible */ + op += cSize; + /* check compressibility */ + assert(op >= ostart); + if ((size_t)(op-ostart) >= srcSize-1) { return 0; } + return (size_t)(op-ostart); +} + +typedef struct { + unsigned count[HUF_SYMBOLVALUE_MAX + 1]; + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; + union { + HUF_buildCTable_wksp_tables buildCTable_wksp; + HUF_WriteCTableWksp writeCTable_wksp; + U32 hist_wksp[HIST_WKSP_SIZE_U32]; + } wksps; +} HUF_compress_tables_t; + +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ + +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue) +{ + unsigned cardinality = 0; + unsigned i; + + for (i = 0; i < maxSymbolValue + 1; i++) { + if (count[i] != 0) cardinality += 1; + } + + return cardinality; +} + +unsigned HUF_minTableLog(unsigned symbolCardinality) +{ + U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1; + return minBitsSymbols; +} + +unsigned HUF_optimalTableLog( + unsigned maxTableLog, + size_t srcSize, + unsigned maxSymbolValue, + void* workSpace, size_t wkspSize, + HUF_CElt* table, + const unsigned* count, + int flags) +{ + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables)); + + if (!(flags & HUF_flags_optimalDepth)) { + /* cheap evaluation, based on FSE */ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + } + + { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); + size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); + size_t maxBits, hSize, newSize; + const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); + const unsigned minTableLog = HUF_minTableLog(symbolCardinality); + size_t optSize = ((size_t) ~0) - 1; + unsigned optLog = maxTableLog, optLogGuess; + + DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); + + /* Search until size increases */ + for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { + DEBUGLOG(7, "checking for huffLog=%u", optLogGuess); + maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); + if (ERR_isError(maxBits)) continue; + + if (maxBits < optLogGuess && optLogGuess > minTableLog) break; + + hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize); + + if (ERR_isError(hSize)) continue; + + newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; + + if (newSize > optSize + 1) { + break; + } + + if (newSize < optSize) { + optSize = newSize; + optLog = optLogGuess; + } + } + assert(optLog <= HUF_TABLELOG_MAX); + return optLog; + } +} + +/* HUF_compress_internal() : + * `workSpace_align4` must be aligned on 4-bytes boundaries, + * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ +static size_t +HUF_compress_internal (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + HUF_nbStreams_e nbStreams, + void* workSpace, size_t wkspSize, + HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags) +{ + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize); + HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); + + /* checks & inits */ + if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall); + if (!srcSize) return 0; /* Uncompressed */ + if (!dstSize) return 0; /* cannot fit anything within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Heuristic : If old table is valid, use it for small inputs */ + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } + + /* If uncompressible data is suspected, do a smaller sampling first */ + DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); + if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { + size_t largestTotal = 0; + DEBUGLOG(5, "input suspected incompressible : sampling to check"); + { unsigned maxSymbolValueBegin = maxSymbolValue; + CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestBegin; + } + { unsigned maxSymbolValueEnd = maxSymbolValue; + CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestEnd; + } + if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1)); + + /* Check validity of previous table */ + if ( repeat + && *repeat == HUF_repeat_check + && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { + *repeat = HUF_repeat_none; + } + /* Heuristic : use existing table for small inputs */ + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags); + { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, + maxSymbolValue, huffLog, + &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); + CHECK_F(maxBits); + huffLog = (U32)maxBits; + DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1)); + } + /* Zero unused symbols in CTable, so we can check it for validity */ + { + size_t const ctableSize = HUF_CTABLE_SIZE_ST(maxSymbolValue); + size_t const unusedSize = sizeof(table->CTable) - ctableSize * sizeof(HUF_CElt); + ZSTD_memset(table->CTable + ctableSize, 0, unusedSize); + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog, + &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) ); + /* Check if using previous huffman table is beneficial */ + if (repeat && *repeat != HUF_repeat_none) { + size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); + size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); + if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } } + + /* Use the new huffman table */ + if (hSize + 12ul >= srcSize) { return 0; } + op += hSize; + if (repeat) { *repeat = HUF_repeat_none; } + if (oldHufTable) + ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ + } + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, table->CTable, flags); +} + +size_t HUF_compress1X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) +{ + DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize); + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_singleStream, + workSpace, wkspSize, hufTable, + repeat, flags); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * consider skipping quickly + * re-use an existing huffman compression table */ +size_t HUF_compress4X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) +{ + DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize); + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_fourStreams, + workSpace, wkspSize, + hufTable, repeat, flags); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress.c new file mode 100644 index 000000000..d6133e70b --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress.c @@ -0,0 +1,7032 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ +#include "../common/mem.h" +#include "hist.h" /* HIST_countFast_wksp */ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "../common/fse.h" +#include "../common/huf.h" +#include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" +#include "zstd_fast.h" +#include "zstd_double_fast.h" +#include "zstd_lazy.h" +#include "zstd_opt.h" +#include "zstd_ldm.h" +#include "zstd_compress_superblock.h" +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_rotateRight_U64 */ + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * COMPRESS_HEAPMODE : + * Select how default decompression function ZSTD_compress() allocates its context, + * on stack (0, default), or into heap (1). + * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. + */ +#ifndef ZSTD_COMPRESS_HEAPMODE +# define ZSTD_COMPRESS_HEAPMODE 0 +#endif + +/*! + * ZSTD_HASHLOG3_MAX : + * Maximum size of the hash table dedicated to find 3-bytes matches, + * in log format, aka 17 => 1 << 17 == 128Ki positions. + * This structure is only used in zstd_opt. + * Since allocation is centralized for all strategies, it has to be known here. + * The actual (selected) size of the hash table is then stored in ZSTD_matchState_t.hashLog3, + * so that zstd_opt.c doesn't need to know about this constant. + */ +#ifndef ZSTD_HASHLOG3_MAX +# define ZSTD_HASHLOG3_MAX 17 +#endif + +/*-************************************* +* Helper functions +***************************************/ +/* ZSTD_compressBound() + * Note that the result from this function is only valid for + * the one-pass compression functions. + * When employing the streaming mode, + * if flushes are frequently altering the size of blocks, + * the overhead from block headers can make the compressed data larger + * than the return value of ZSTD_compressBound(). + */ +size_t ZSTD_compressBound(size_t srcSize) { + size_t const r = ZSTD_COMPRESSBOUND(srcSize); + if (r==0) return ERROR(srcSize_wrong); + return r; +} + + +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CDict_s { + const void* dictContent; + size_t dictContentSize; + ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_cwksp workspace; + ZSTD_matchState_t matchState; + ZSTD_compressedBlockState_t cBlockState; + ZSTD_customMem customMem; + U32 dictID; + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use + * row-based matchfinder. Unless the cdict is reloaded, we will use + * the same greedy/lazy matchfinder at compression time. + */ +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); +} + +static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) +{ + assert(cctx != NULL); + ZSTD_memset(cctx, 0, sizeof(*cctx)); + cctx->customMem = memManager; + cctx->bmi2 = ZSTD_cpuSupportsBmi2(); + { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_STATIC_ASSERT(zcss_init==0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + ZSTD_initCCtx(cctx, customMem); + return cctx; + } +} + +ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) +{ + ZSTD_cwksp ws; + ZSTD_CCtx* cctx; + if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ + if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); + if (cctx == NULL) return NULL; + + ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_cwksp_move(&cctx->workspace, &ws); + cctx->staticSize = workspaceSize; + + /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ + if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + return cctx; +} + +/** + * Clears and frees all of the dictionaries in the CCtx. + */ +static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) +{ + ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); + ZSTD_freeCDict(cctx->localDict.cdict); + ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); + cctx->cdict = NULL; +} + +static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) +{ + size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; + size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); + return bufferSize + cdictSize; +} + +static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) +{ + assert(cctx != NULL); + assert(cctx->staticSize == 0); + ZSTD_clearAllDicts(cctx); +#ifdef ZSTD_MULTITHREAD + ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; +#endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "not compatible with static CCtx"); + { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem); + } + return 0; +} + + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_sizeof_CCtx(cctx->mtctx); +#else + (void)cctx; + return 0; +#endif +} + + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + + ZSTD_sizeof_localDict(cctx->localDict) + + ZSTD_sizeof_mtctx(cctx); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + +/* Returns true if the strategy supports using a row based matchfinder */ +static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { + return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2); +} + +/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder + * for this compression. + */ +static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) { + assert(mode != ZSTD_ps_auto); + return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable); +} + +/* Returns row matchfinder usage given an initial mode and cParams */ +static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { +#if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON) + int const kHasSIMD128 = 1; +#else + int const kHasSIMD128 = 0; +#endif + if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ + mode = ZSTD_ps_disable; + if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; + if (kHasSIMD128) { + if (cParams->windowLog > 14) mode = ZSTD_ps_enable; + } else { + if (cParams->windowLog > 17) mode = ZSTD_ps_enable; + } + return mode; +} + +/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */ +static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ +static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, + const ZSTD_paramSwitch_e useRowMatchFinder, + const U32 forDDSDict) { + assert(useRowMatchFinder != ZSTD_ps_auto); + /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. + * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. + */ + return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); +} + +/* Returns ZSTD_ps_enable if compression parameters are such that we should + * enable long distance matching (wlog >= 27, strategy >= btopt). + * Returns ZSTD_ps_disable otherwise. + */ +static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +static int ZSTD_resolveExternalSequenceValidation(int mode) { + return mode; +} + +/* Resolves maxBlockSize to the default if no value is present. */ +static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { + if (maxBlockSize == 0) { + return ZSTD_BLOCKSIZE_MAX; + } else { + return maxBlockSize; + } +} + +static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) { + if (value != ZSTD_ps_auto) return value; + if (cLevel < 10) { + return ZSTD_ps_disable; + } else { + return ZSTD_ps_enable; + } +} + +/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged. + * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */ +static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) { + return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast; +} + +static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( + ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params cctxParams; + /* should not matter, as all cParams are presumed properly defined */ + ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); + cctxParams.cParams = cParams; + + /* Adjust advanced params according to cParams */ + cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams); + if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); + assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); + assert(cctxParams.ldmParams.hashRateLog < 32); + } + cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams); + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences); + cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize); + cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes, + cctxParams.compressionLevel); + assert(!ZSTD_checkCParams(cParams)); + return cctxParams; +} + +static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params* params; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + params = (ZSTD_CCtx_params*)ZSTD_customCalloc( + sizeof(ZSTD_CCtx_params), customMem); + if (!params) { return NULL; } + ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); + params->customMem = customMem; + return params; +} + +ZSTD_CCtx_params* ZSTD_createCCtxParams(void) +{ + return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) +{ + if (params == NULL) { return 0; } + ZSTD_customFree(params, params->customMem); + return 0; +} + +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) +{ + return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); +} + +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->compressionLevel = compressionLevel; + cctxParams->fParams.contentSizeFlag = 1; + return 0; +} + +#define ZSTD_NO_CLEVEL 0 + +/** + * Initializes `cctxParams` from `params` and `compressionLevel`. + * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. + */ +static void +ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, + const ZSTD_parameters* params, + int compressionLevel) +{ + assert(!ZSTD_checkCParams(params->cParams)); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = compressionLevel; + cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); + cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams); + cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); + cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences); + cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); + cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel); + DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", + cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm); +} + +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +{ + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return 0; +} + +/** + * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. + * @param params Validated zstd parameters. + */ +static void ZSTD_CCtxParams_setZstdParams( + ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) +{ + assert(!ZSTD_checkCParams(params->cParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = ZSTD_NO_CLEVEL; +} + +ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + + switch(param) + { + case ZSTD_c_compressionLevel: + bounds.lowerBound = ZSTD_minCLevel(); + bounds.upperBound = ZSTD_maxCLevel(); + return bounds; + + case ZSTD_c_windowLog: + bounds.lowerBound = ZSTD_WINDOWLOG_MIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + + case ZSTD_c_hashLog: + bounds.lowerBound = ZSTD_HASHLOG_MIN; + bounds.upperBound = ZSTD_HASHLOG_MAX; + return bounds; + + case ZSTD_c_chainLog: + bounds.lowerBound = ZSTD_CHAINLOG_MIN; + bounds.upperBound = ZSTD_CHAINLOG_MAX; + return bounds; + + case ZSTD_c_searchLog: + bounds.lowerBound = ZSTD_SEARCHLOG_MIN; + bounds.upperBound = ZSTD_SEARCHLOG_MAX; + return bounds; + + case ZSTD_c_minMatch: + bounds.lowerBound = ZSTD_MINMATCH_MIN; + bounds.upperBound = ZSTD_MINMATCH_MAX; + return bounds; + + case ZSTD_c_targetLength: + bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; + bounds.upperBound = ZSTD_TARGETLENGTH_MAX; + return bounds; + + case ZSTD_c_strategy: + bounds.lowerBound = ZSTD_STRATEGY_MIN; + bounds.upperBound = ZSTD_STRATEGY_MAX; + return bounds; + + case ZSTD_c_contentSizeFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_checksumFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_dictIDFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_nbWorkers: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_NBWORKERS_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_jobSize: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_JOBSIZE_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_overlapLog: +#ifdef ZSTD_MULTITHREAD + bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; + bounds.upperBound = ZSTD_OVERLAPLOG_MAX; +#else + bounds.lowerBound = 0; + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_enableDedicatedDictSearch: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_enableLongDistanceMatching: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; + + case ZSTD_c_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; + + case ZSTD_c_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; + + case ZSTD_c_ldmHashRateLog: + bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; + return bounds; + + /* experimental parameters */ + case ZSTD_c_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_forceMaxWindow : + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_format: + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + bounds.lowerBound = ZSTD_f_zstd1; + bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_forceAttachDict: + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); + bounds.lowerBound = ZSTD_dictDefaultAttach; + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_literalCompressionMode: + ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable); + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_srcSizeHint: + bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; + bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; + return bounds; + + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + + case ZSTD_c_blockDelimiters: + bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; + bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; + return bounds; + + case ZSTD_c_validateSequences: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_useBlockSplitter: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_useRowMatchFinder: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_deterministicRefPrefix: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_prefetchCDictTables: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_enableSeqProducerFallback: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_searchForExternalRepcodes: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + default: + bounds.error = ERROR(parameter_unsupported); + return bounds; + } +} + +/* ZSTD_cParam_clampBounds: + * Clamps the value into the bounded range. + */ +static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return bounds.error; + if (*value < bounds.lowerBound) *value = bounds.lowerBound; + if (*value > bounds.upperBound) *value = bounds.upperBound; + return 0; +} + +#define BOUNDCHECK(cParam, val) { \ + RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ + parameter_outOfBound, "Param out of bounds"); \ +} + + +static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) +{ + switch(param) + { + case ZSTD_c_compressionLevel: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + return 1; + + case ZSTD_c_format: + case ZSTD_c_windowLog: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow : + case ZSTD_c_nbWorkers: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_useBlockSplitter: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_searchForExternalRepcodes: + default: + return 0; + } +} + +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); + if (cctx->streamStage != zcss_init) { + if (ZSTD_isUpdateAuthorized(param)) { + cctx->cParamsChanged = 1; + } else { + RETURN_ERROR(stage_wrong, "can only set params in cctx init stage"); + } } + + switch(param) + { + case ZSTD_c_nbWorkers: + RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, + "MT not compatible with static alloc"); + break; + + case ZSTD_c_compressionLevel: + case ZSTD_c_windowLog: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_format: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_useBlockSplitter: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_searchForExternalRepcodes: + break; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, + ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); + switch(param) + { + case ZSTD_c_format : + BOUNDCHECK(ZSTD_c_format, value); + CCtxParams->format = (ZSTD_format_e)value; + return (size_t)CCtxParams->format; + + case ZSTD_c_compressionLevel : { + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + if (value == 0) + CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else + CCtxParams->compressionLevel = value; + if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; + return 0; /* return type (size_t) cannot represent negative values */ + } + + case ZSTD_c_windowLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_windowLog, value); + CCtxParams->cParams.windowLog = (U32)value; + return CCtxParams->cParams.windowLog; + + case ZSTD_c_hashLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_hashLog, value); + CCtxParams->cParams.hashLog = (U32)value; + return CCtxParams->cParams.hashLog; + + case ZSTD_c_chainLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_chainLog, value); + CCtxParams->cParams.chainLog = (U32)value; + return CCtxParams->cParams.chainLog; + + case ZSTD_c_searchLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_searchLog, value); + CCtxParams->cParams.searchLog = (U32)value; + return (size_t)value; + + case ZSTD_c_minMatch : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_minMatch, value); + CCtxParams->cParams.minMatch = (U32)value; + return CCtxParams->cParams.minMatch; + + case ZSTD_c_targetLength : + BOUNDCHECK(ZSTD_c_targetLength, value); + CCtxParams->cParams.targetLength = (U32)value; + return CCtxParams->cParams.targetLength; + + case ZSTD_c_strategy : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_strategy, value); + CCtxParams->cParams.strategy = (ZSTD_strategy)value; + return (size_t)CCtxParams->cParams.strategy; + + case ZSTD_c_contentSizeFlag : + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(4, "set content size flag = %u", (value!=0)); + CCtxParams->fParams.contentSizeFlag = value != 0; + return (size_t)CCtxParams->fParams.contentSizeFlag; + + case ZSTD_c_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + CCtxParams->fParams.checksumFlag = value != 0; + return (size_t)CCtxParams->fParams.checksumFlag; + + case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); + CCtxParams->fParams.noDictIDFlag = !value; + return !CCtxParams->fParams.noDictIDFlag; + + case ZSTD_c_forceMaxWindow : + CCtxParams->forceWindow = (value != 0); + return (size_t)CCtxParams->forceWindow; + + case ZSTD_c_forceAttachDict : { + const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; + BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref); + CCtxParams->attachDictPref = pref; + return CCtxParams->attachDictPref; + } + + case ZSTD_c_literalCompressionMode : { + const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value; + BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm); + CCtxParams->literalCompressionMode = lcm; + return CCtxParams->literalCompressionMode; + } + + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + CCtxParams->nbWorkers = value; + return CCtxParams->nbWorkers; +#endif + + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + /* Adjust to the minimum non-default value. */ + if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) + value = ZSTDMT_JOBSIZE_MIN; + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + assert(value >= 0); + CCtxParams->jobSize = value; + return CCtxParams->jobSize; +#endif + + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->overlapLog = value; + return CCtxParams->overlapLog; +#endif + + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->rsyncable = value; + return CCtxParams->rsyncable; +#endif + + case ZSTD_c_enableDedicatedDictSearch : + CCtxParams->enableDedicatedDictSearch = (value!=0); + return (size_t)CCtxParams->enableDedicatedDictSearch; + + case ZSTD_c_enableLongDistanceMatching : + BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value); + CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value; + return CCtxParams->ldmParams.enableLdm; + + case ZSTD_c_ldmHashLog : + if (value!=0) /* 0 ==> auto */ + BOUNDCHECK(ZSTD_c_ldmHashLog, value); + CCtxParams->ldmParams.hashLog = (U32)value; + return CCtxParams->ldmParams.hashLog; + + case ZSTD_c_ldmMinMatch : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmMinMatch, value); + CCtxParams->ldmParams.minMatchLength = (U32)value; + return CCtxParams->ldmParams.minMatchLength; + + case ZSTD_c_ldmBucketSizeLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); + CCtxParams->ldmParams.bucketSizeLog = (U32)value; + return CCtxParams->ldmParams.bucketSizeLog; + + case ZSTD_c_ldmHashRateLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmHashRateLog, value); + CCtxParams->ldmParams.hashRateLog = (U32)value; + return CCtxParams->ldmParams.hashRateLog; + + case ZSTD_c_targetCBlockSize : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + CCtxParams->targetCBlockSize = (U32)value; + return CCtxParams->targetCBlockSize; + + case ZSTD_c_srcSizeHint : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_srcSizeHint, value); + CCtxParams->srcSizeHint = value; + return (size_t)CCtxParams->srcSizeHint; + + case ZSTD_c_stableInBuffer: + BOUNDCHECK(ZSTD_c_stableInBuffer, value); + CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->inBufferMode; + + case ZSTD_c_stableOutBuffer: + BOUNDCHECK(ZSTD_c_stableOutBuffer, value); + CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->outBufferMode; + + case ZSTD_c_blockDelimiters: + BOUNDCHECK(ZSTD_c_blockDelimiters, value); + CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; + return CCtxParams->blockDelimiters; + + case ZSTD_c_validateSequences: + BOUNDCHECK(ZSTD_c_validateSequences, value); + CCtxParams->validateSequences = value; + return CCtxParams->validateSequences; + + case ZSTD_c_useBlockSplitter: + BOUNDCHECK(ZSTD_c_useBlockSplitter, value); + CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value; + return CCtxParams->useBlockSplitter; + + case ZSTD_c_useRowMatchFinder: + BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); + CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value; + return CCtxParams->useRowMatchFinder; + + case ZSTD_c_deterministicRefPrefix: + BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); + CCtxParams->deterministicRefPrefix = !!value; + return CCtxParams->deterministicRefPrefix; + + case ZSTD_c_prefetchCDictTables: + BOUNDCHECK(ZSTD_c_prefetchCDictTables, value); + CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value; + return CCtxParams->prefetchCDictTables; + + case ZSTD_c_enableSeqProducerFallback: + BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value); + CCtxParams->enableMatchFinderFallback = value; + return CCtxParams->enableMatchFinderFallback; + + case ZSTD_c_maxBlockSize: + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_maxBlockSize, value); + CCtxParams->maxBlockSize = value; + return CCtxParams->maxBlockSize; + + case ZSTD_c_searchForExternalRepcodes: + BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value); + CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value; + return CCtxParams->searchForExternalRepcodes; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } +} + +size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) +{ + return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_getParameter( + ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) +{ + switch(param) + { + case ZSTD_c_format : + *value = CCtxParams->format; + break; + case ZSTD_c_compressionLevel : + *value = CCtxParams->compressionLevel; + break; + case ZSTD_c_windowLog : + *value = (int)CCtxParams->cParams.windowLog; + break; + case ZSTD_c_hashLog : + *value = (int)CCtxParams->cParams.hashLog; + break; + case ZSTD_c_chainLog : + *value = (int)CCtxParams->cParams.chainLog; + break; + case ZSTD_c_searchLog : + *value = CCtxParams->cParams.searchLog; + break; + case ZSTD_c_minMatch : + *value = CCtxParams->cParams.minMatch; + break; + case ZSTD_c_targetLength : + *value = CCtxParams->cParams.targetLength; + break; + case ZSTD_c_strategy : + *value = (unsigned)CCtxParams->cParams.strategy; + break; + case ZSTD_c_contentSizeFlag : + *value = CCtxParams->fParams.contentSizeFlag; + break; + case ZSTD_c_checksumFlag : + *value = CCtxParams->fParams.checksumFlag; + break; + case ZSTD_c_dictIDFlag : + *value = !CCtxParams->fParams.noDictIDFlag; + break; + case ZSTD_c_forceMaxWindow : + *value = CCtxParams->forceWindow; + break; + case ZSTD_c_forceAttachDict : + *value = CCtxParams->attachDictPref; + break; + case ZSTD_c_literalCompressionMode : + *value = CCtxParams->literalCompressionMode; + break; + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + assert(CCtxParams->nbWorkers == 0); +#endif + *value = CCtxParams->nbWorkers; + break; + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + assert(CCtxParams->jobSize <= INT_MAX); + *value = (int)CCtxParams->jobSize; + break; +#endif + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->overlapLog; + break; +#endif + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->rsyncable; + break; +#endif + case ZSTD_c_enableDedicatedDictSearch : + *value = CCtxParams->enableDedicatedDictSearch; + break; + case ZSTD_c_enableLongDistanceMatching : + *value = CCtxParams->ldmParams.enableLdm; + break; + case ZSTD_c_ldmHashLog : + *value = CCtxParams->ldmParams.hashLog; + break; + case ZSTD_c_ldmMinMatch : + *value = CCtxParams->ldmParams.minMatchLength; + break; + case ZSTD_c_ldmBucketSizeLog : + *value = CCtxParams->ldmParams.bucketSizeLog; + break; + case ZSTD_c_ldmHashRateLog : + *value = CCtxParams->ldmParams.hashRateLog; + break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; + case ZSTD_c_srcSizeHint : + *value = (int)CCtxParams->srcSizeHint; + break; + case ZSTD_c_stableInBuffer : + *value = (int)CCtxParams->inBufferMode; + break; + case ZSTD_c_stableOutBuffer : + *value = (int)CCtxParams->outBufferMode; + break; + case ZSTD_c_blockDelimiters : + *value = (int)CCtxParams->blockDelimiters; + break; + case ZSTD_c_validateSequences : + *value = (int)CCtxParams->validateSequences; + break; + case ZSTD_c_useBlockSplitter : + *value = (int)CCtxParams->useBlockSplitter; + break; + case ZSTD_c_useRowMatchFinder : + *value = (int)CCtxParams->useRowMatchFinder; + break; + case ZSTD_c_deterministicRefPrefix: + *value = (int)CCtxParams->deterministicRefPrefix; + break; + case ZSTD_c_prefetchCDictTables: + *value = (int)CCtxParams->prefetchCDictTables; + break; + case ZSTD_c_enableSeqProducerFallback: + *value = CCtxParams->enableMatchFinderFallback; + break; + case ZSTD_c_maxBlockSize: + *value = (int)CCtxParams->maxBlockSize; + break; + case ZSTD_c_searchForExternalRepcodes: + *value = (int)CCtxParams->searchForExternalRepcodes; + break; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return 0; +} + +/** ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. + * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. + * This is possible even if a compression is ongoing. + * In which case, new parameters will be applied on the fly, starting with next compression job. + */ +size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "The context is in the wrong stage!"); + RETURN_ERROR_IF(cctx->cdict, stage_wrong, + "Can't override parameters with cdict attached (some must " + "be inherited from the cdict)."); + + cctx->requestedParams = *params; + return 0; +} + +size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams) +{ + ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setCParams"); + /* only update if all parameters are valid */ + FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), ""); + return 0; +} + +size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams) +{ + ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setFParams"); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), ""); + return 0; +} + +size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParams"); + /* First check cParams, because we want to update all or none. */ + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + /* Next set fParams, because this could fail if the cctx isn't in init stage. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), ""); + /* Finally set cParams, which should succeed. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), ""); + return 0; +} + +size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't set pledgedSrcSize when not in init stage."); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + return 0; +} + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( + int const compressionLevel, + size_t const dictSize); +static int ZSTD_dedicatedDictSearch_isSupported( + const ZSTD_compressionParameters* cParams); +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams); + +/** + * Initializes the local dictionary using requested parameters. + * NOTE: Initialization does not employ the pledged src size, + * because the dictionary may be used for multiple compressions. + */ +static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) +{ + ZSTD_localDict* const dl = &cctx->localDict; + if (dl->dict == NULL) { + /* No local dictionary. */ + assert(dl->dictBuffer == NULL); + assert(dl->cdict == NULL); + assert(dl->dictSize == 0); + return 0; + } + if (dl->cdict != NULL) { + /* Local dictionary already initialized. */ + assert(cctx->cdict == dl->cdict); + return 0; + } + assert(dl->dictSize > 0); + assert(cctx->cdict == NULL); + assert(cctx->prefixDict.dict == NULL); + + dl->cdict = ZSTD_createCDict_advanced2( + dl->dict, + dl->dictSize, + ZSTD_dlm_byRef, + dl->dictContentType, + &cctx->requestedParams, + cctx->customMem); + RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); + cctx->cdict = dl->cdict; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_advanced( + ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when cctx is not in init stage."); + ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */ + if (dict == NULL || dictSize == 0) /* no dictionary */ + return 0; + if (dictLoadMethod == ZSTD_dlm_byRef) { + cctx->localDict.dict = dict; + } else { + /* copy dictionary content inside CCtx to own its lifetime */ + void* dictBuffer; + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "static CCtx can't allocate for an internal copy of dictionary"); + dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); + RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation, + "allocation failed for dictionary content"); + ZSTD_memcpy(dictBuffer, dict, dictSize); + cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */ + cctx->localDict.dict = dictBuffer; /* read-only reference */ + } + cctx->localDict.dictSize = dictSize; + cctx->localDict.dictContentType = dictContentType; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_byReference( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + + +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a dict when ctx not in init stage."); + /* Free the existing local cdict (if any) to save memory. */ + ZSTD_clearAllDicts(cctx); + cctx->cdict = cdict; + return 0; +} + +size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a pool when ctx not in init stage."); + cctx->pool = pool; + return 0; +} + +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a prefix when ctx not in init stage."); + ZSTD_clearAllDicts(cctx); + if (prefix != NULL && prefixSize > 0) { + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictContentType = dictContentType; + } + return 0; +} + +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Reset parameters is only possible during init stage."); + ZSTD_clearAllDicts(cctx); + ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx)); + return ZSTD_CCtxParams_reset(&cctx->requestedParams); + } + return 0; +} + + +/** ZSTD_checkCParams() : + control CParam values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); + BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); + return 0; +} + +/** ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters +ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMP_TYPE(cParam, val, type) { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ + } +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) + CLAMP(ZSTD_c_windowLog, cParams.windowLog); + CLAMP(ZSTD_c_chainLog, cParams.chainLog); + CLAMP(ZSTD_c_hashLog, cParams.hashLog); + CLAMP(ZSTD_c_searchLog, cParams.searchLog); + CLAMP(ZSTD_c_minMatch, cParams.minMatch); + CLAMP(ZSTD_c_targetLength,cParams.targetLength); + CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); + return cParams; +} + +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + +/** ZSTD_dictAndWindowLog() : + * Returns an adjusted window log that is large enough to fit the source and the dictionary. + * The zstd format says that the entire dictionary is valid if one byte of the dictionary + * is within the window. So the hashLog and chainLog should be large enough to reference both + * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing + * the hashLog and windowLog. + * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. + */ +static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) +{ + const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; + /* No dictionary ==> No change */ + if (dictSize == 0) { + return windowLog; + } + assert(windowLog <= ZSTD_WINDOWLOG_MAX); + assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ + { + U64 const windowSize = 1ULL << windowLog; + U64 const dictAndWindowSize = dictSize + windowSize; + /* If the window size is already large enough to fit both the source and the dictionary + * then just use the window size. Otherwise adjust so that it fits the dictionary and + * the window. + */ + if (windowSize >= dictSize + srcSize) { + return windowLog; /* Window size large enough already */ + } else if (dictAndWindowSize >= maxWindowSize) { + return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ + } else { + return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; + } + } +} + +/** ZSTD_adjustCParams_internal() : + * optimize `cPar` for a specified input (`srcSize` and `dictSize`). + * mostly downsize to reduce memory consumption and initialization latency. + * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. + * note : `srcSize==0` means 0! + * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ +static ZSTD_compressionParameters +ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize, + ZSTD_cParamMode_e mode, + ZSTD_paramSwitch_e useRowMatchFinder) +{ + const U64 minSrcSize = 513; /* (1<<9) + 1 */ + const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); + assert(ZSTD_checkCParams(cPar)==0); + + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + /* If we don't know the source size, don't make any + * assumptions about it. We will already have selected + * smaller parameters if a dictionary is in use. + */ + break; + case ZSTD_cpm_createCDict: + /* Assume a small source size when creating a dictionary + * with an unknown source size. + */ + if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) + srcSize = minSrcSize; + break; + case ZSTD_cpm_attachDict: + /* Dictionary has its own dedicated parameters which have + * already been selected. We are selecting parameters + * for only the source. + */ + dictSize = 0; + break; + default: + assert(0); + break; + } + + /* resize windowLog if input is small enough, to use less memory */ + if ( (srcSize <= maxWindowResize) + && (dictSize <= maxWindowResize) ) { + U32 const tSize = (U32)(srcSize + dictSize); + static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; + U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : + ZSTD_highbit32(tSize-1) + 1; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; + } + if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); + U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; + if (cycleLog > dictAndWindowLog) + cPar.chainLog -= (cycleLog - dictAndWindowLog); + } + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ + + /* We can't use more than 32 bits of hash in total, so that means that we require: + * (hashLog + 8) <= 32 && (chainLog + 8) <= 32 + */ + if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) { + U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; + if (cPar.hashLog > maxShortCacheHashLog) { + cPar.hashLog = maxShortCacheHashLog; + } + if (cPar.chainLog > maxShortCacheHashLog) { + cPar.chainLog = maxShortCacheHashLog; + } + } + + + /* At this point, we aren't 100% sure if we are using the row match finder. + * Unless it is explicitly disabled, conservatively assume that it is enabled. + * In this case it will only be disabled for small sources, so shrinking the + * hash log a little bit shouldn't result in any ratio loss. + */ + if (useRowMatchFinder == ZSTD_ps_auto) + useRowMatchFinder = ZSTD_ps_enable; + + /* We can't hash more than 32-bits in total. So that means that we require: + * (hashLog - rowLog + 8) <= 32 + */ + if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) { + /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cPar.searchLog, 6); + U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; + U32 const maxHashLog = maxRowHashLog + rowLog; + assert(cPar.hashLog >= rowLog); + if (cPar.hashLog > maxHashLog) { + cPar.hashLog = maxHashLog; + } + } + + return cPar; +} + +ZSTD_compressionParameters +ZSTD_adjustCParams(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ + if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto); +} + +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +static void ZSTD_overrideCParams( + ZSTD_compressionParameters* cParams, + const ZSTD_compressionParameters* overrides) +{ + if (overrides->windowLog) cParams->windowLog = overrides->windowLog; + if (overrides->hashLog) cParams->hashLog = overrides->hashLog; + if (overrides->chainLog) cParams->chainLog = overrides->chainLog; + if (overrides->searchLog) cParams->searchLog = overrides->searchLog; + if (overrides->minMatch) cParams->minMatch = overrides->minMatch; + if (overrides->targetLength) cParams->targetLength = overrides->targetLength; + if (overrides->strategy) cParams->strategy = overrides->strategy; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + ZSTD_compressionParameters cParams; + if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { + srcSizeHint = CCtxParams->srcSizeHint; + } + cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); + if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); + assert(!ZSTD_checkCParams(cParams)); + /* srcSizeHint == 0 means 0 */ + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder); +} + +static size_t +ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + const ZSTD_paramSwitch_e useRowMatchFinder, + const U32 enableDedicatedDictSearch, + const U32 forCCtx) +{ + /* chain table size should be 0 for fast or row-hash strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); + size_t const optPotentialSpace = + ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((1<strategy, useRowMatchFinder) + ? ZSTD_cwksp_aligned_alloc_size(hSize) + : 0; + size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) + ? optPotentialSpace + : 0; + size_t const slackSpace = ZSTD_cwksp_slack_space_required(); + + /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ + ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); + assert(useRowMatchFinder != ZSTD_ps_auto); + + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", + (U32)chainSize, (U32)hSize, (U32)h3Size); + return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; +} + +/* Helper function for calculating memory requirements. + * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ +static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { + U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4; + return blockSize / divider; +} + +static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( + const ZSTD_compressionParameters* cParams, + const ldmParams_t* ldmParams, + const int isStatic, + const ZSTD_paramSwitch_e useRowMatchFinder, + const size_t buffInSize, + const size_t buffOutSize, + const U64 pledgedSrcSize, + int useSequenceProducer, + size_t maxBlockSize) +{ + size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); + + size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); + size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? + ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; + + + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + + ZSTD_cwksp_alloc_size(buffOutSize); + + size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + size_t const externalSeqSpace = useSequenceProducer + ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) + : 0; + + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace + + externalSeqSpace; + + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return neededSpace; +} + +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, + &cParams); + + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + /* estimateCCtxSize is for one-shot compression. So no buffers should + * be needed. However, we still allocate two 0-sized buffers, which can + * take space under ASAN. */ + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize); +} + +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + } +} + +static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) +{ + int tier = 0; + size_t largestSize = 0; + static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; + for (; tier < 4; ++tier) { + /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); + largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); + } + return largestSize; +} + +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + /* Ensure monotonically increasing memory usage as compression level increases */ + size_t const newMB = ZSTD_estimateCCtxSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + { ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog); + size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) + ? ((size_t)1 << cParams.windowLog) + blockSize + : 0; + size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); + + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, + ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize); + } +} + +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + } +} + +static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + return ZSTD_estimateCStreamSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + size_t const newMB = ZSTD_estimateCStreamSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +/* ZSTD_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads (non-blocking mode). + */ +ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_getFrameProgression(cctx->mtctx); + } +#endif + { ZSTD_frameProgression fp; + size_t const buffered = (cctx->inBuff == NULL) ? 0 : + cctx->inBuffPos - cctx->inToCompress; + if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); + assert(buffered <= ZSTD_BLOCKSIZE_MAX); + fp.ingested = cctx->consumedSrcSize + buffered; + fp.consumed = cctx->consumedSrcSize; + fp.produced = cctx->producedCSize; + fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ + fp.currentJobID = 0; + fp.nbActiveWorkers = 0; + return fp; +} } + +/*! ZSTD_toFlushNow() + * Only useful for multithreading scenarios currently (nbWorkers >= 1). + */ +size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_toFlushNow(cctx->mtctx); + } +#endif + (void)cctx; + return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ +} + +static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, + ZSTD_compressionParameters cParams2) +{ + (void)cParams1; + (void)cParams2; + assert(cParams1.windowLog == cParams2.windowLog); + assert(cParams1.chainLog == cParams2.chainLog); + assert(cParams1.hashLog == cParams2.hashLog); + assert(cParams1.searchLog == cParams2.searchLog); + assert(cParams1.minMatch == cParams2.minMatch); + assert(cParams1.targetLength == cParams2.targetLength); + assert(cParams1.strategy == cParams2.strategy); +} + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) +{ + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + bs->rep[i] = repStartValue[i]; + bs->entropy.huf.repeatMode = HUF_repeat_none; + bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; + bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; + bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; +} + +/*! ZSTD_invalidateMatchState() + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). + */ +static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) +{ + ZSTD_window_clear(&ms->window); + + ms->nextToUpdate = ms->window.dictLimit; + ms->loadedDictEnd = 0; + ms->opt.litLengthSum = 0; /* force reset of btopt stats */ + ms->dictMatchState = NULL; +} + +/** + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ +typedef enum { + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty +} ZSTD_compResetPolicy_e; + +/** + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). + */ +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; + +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; + +/* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */ +static U64 ZSTD_bitmix(U64 val, U64 len) { + val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24); + val *= 0x9FB21C651E98DF25ULL; + val ^= (val >> 35) + len ; + val *= 0x9FB21C651E98DF25ULL; + return val ^ (val >> 28); +} + +/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */ +static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) { + ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4); +} + +static size_t +ZSTD_reset_matchState(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + const ZSTD_compressionParameters* cParams, + const ZSTD_paramSwitch_e useRowMatchFinder, + const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, + const ZSTD_resetTarget_e forWho) +{ + /* disable chain table allocation for fast or row-based strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, + ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + assert(useRowMatchFinder != ZSTD_ps_auto); + if (forceResetIndex == ZSTDirp_reset) { + ZSTD_window_init(&ms->window); + ZSTD_cwksp_mark_tables_dirty(ws); + } + + ms->hashLog3 = hashLog3; + ms->lazySkipping = 0; + + ZSTD_invalidateMatchState(ms); + + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + ZSTD_cwksp_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { + /* reset tables only */ + ZSTD_cwksp_clean_tables(ws); + } + + if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { + /* Row match finder needs an additional table of hashes ("tags") */ + size_t const tagTableSize = hSize; + /* We want to generate a new salt in case we reset a Cctx, but we always want to use + * 0 when we reset a Cdict */ + if(forWho == ZSTD_resetTarget_CCtx) { + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); + ZSTD_advanceHashSalt(ms); + } else { + /* When we are not salting we want to always memset the memory */ + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ms->hashSalt = 0; + } + { /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); + assert(cParams->hashLog >= rowLog); + ms->rowHashLog = cParams->hashLog - rowLog; + } + } + + /* opt parser space */ + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { + DEBUGLOG(4, "reserving optimal parser space"); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); + } + + ms->cParams = *cParams; + + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + return 0; +} + +/* ZSTD_indexTooCloseToMax() : + * minor optimization : prefer memset() rather than reduceIndex() + * which is measurably slow in some circumstances (reported for Visual Studio). + * Works when re-using a context for a lot of smallish inputs : + * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, + * memset() will be triggered before reduceIndex(). + */ +#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) +static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) +{ + return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); +} + +/** ZSTD_dictTooBig(): + * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in + * one go generically. So we ensure that in that case we reset the tables to zero, + * so that we can load as much of the dictionary as possible. + */ +static int ZSTD_dictTooBig(size_t const loadedDictSize) +{ + return loadedDictSize > ZSTD_CHUNKSIZE_MAX; +} + +/*! ZSTD_resetCCtx_internal() : + * @param loadedDictSize The size of the dictionary to be loaded + * into the context, if any. If no dictionary is used, or the + * dictionary is being attached / copied, then pass 0. + * note : `params` are assumed fully validated at this stage. + */ +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, + ZSTD_CCtx_params const* params, + U64 const pledgedSrcSize, + size_t const loadedDictSize, + ZSTD_compResetPolicy_e const crp, + ZSTD_buffered_policy_e const zbuff) +{ + ZSTD_cwksp* const ws = &zc->workspace; + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d", + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + + zc->isFirstBlock = 1; + + /* Set applied params early so we can modify them for LDM, + * and point params at the applied params. + */ + zc->appliedParams = *params; + params = &zc->appliedParams; + + assert(params->useRowMatchFinder != ZSTD_ps_auto); + assert(params->useBlockSplitter != ZSTD_ps_auto); + assert(params->ldmParams.enableLdm != ZSTD_ps_auto); + assert(params->maxBlockSize != 0); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* Adjust long distance matching parameters */ + ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); + assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); + assert(params->ldmParams.hashRateLog < 32); + } + + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(params->maxBlockSize, windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useSequenceProducer); + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) + ? windowSize + blockSize + : 0; + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); + + int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); + ZSTD_indexResetPolicy_e needsIndexReset = + (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; + + size_t const neededSpace = + ZSTD_estimateCCtxSize_usingCCtxParams_internal( + ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, + buffInSize, buffOutSize, pledgedSrcSize, params->useSequenceProducer, params->maxBlockSize); + int resizeWorkspace; + + FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); + + if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); + + { /* Check if workspace is large enough, alloc a new one if needed */ + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); + resizeWorkspace = workspaceTooSmall || workspaceWasteful; + DEBUGLOG(4, "Need %zu B workspace", neededSpace); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + + if (resizeWorkspace) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + ZSTD_cwksp_sizeof(ws) >> 10, + neededSpace >> 10); + + RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); + + needsIndexReset = ZSTDirp_reset; + + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); + + DEBUGLOG(5, "reserving object space"); + /* Statically sized space. + * entropyWorkspace never moves, + * though prev/next block swap places */ + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); + } } + + ZSTD_cwksp_clear(ws); + + /* init params */ + zc->blockState.matchState.cParams = params->cParams; + zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable; + zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; + zc->consumedSrcSize = 0; + zc->producedCSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(4, "pledged content size : %u ; flag : %u", + (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + zc->blockSize = blockSize; + + XXH64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->dictContentSize = 0; + + ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms->cParams, + params->useRowMatchFinder, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx), ""); + + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + + /* ldm hash table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + ZSTD_window_init(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; + } + + /* reserve space for block-level external sequences */ + if (params->useSequenceProducer) { + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq; + zc->externalMatchCtx.seqBuffer = + (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); + } + + /* buffers */ + + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + + zc->bufferedPolicy = zbuff; + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); + + /* ldm bucketOffsets table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const numBuckets = + ((size_t)1) << (params->ldmParams.hashLog - + params->ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); + ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); + } + + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); + assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace)); + + zc->initialized = 1; + + return 0; + } +} + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { + int i; + for (i=0; iblockState.prevCBlock->rep[i] = 0; + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); +} + +/* These are the approximate sizes for each strategy past which copying the + * dictionary tables into the working context is faster than using them + * in-place. + */ +static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { + 8 KB, /* unused */ + 8 KB, /* ZSTD_fast */ + 16 KB, /* ZSTD_dfast */ + 32 KB, /* ZSTD_greedy */ + 32 KB, /* ZSTD_lazy */ + 32 KB, /* ZSTD_lazy2 */ + 32 KB, /* ZSTD_btlazy2 */ + 32 KB, /* ZSTD_btopt */ + 8 KB, /* ZSTD_btultra */ + 8 KB /* ZSTD_btultra2 */ +}; + +static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize) +{ + size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; + int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; + return dedicatedDictSearch + || ( ( pledgedSrcSize <= cutoff + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow ); /* dictMatchState isn't correctly + * handled in _enforceMaxDist */ +} + +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + { + ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; + unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Resize working context table params for input only, since the dict + * has its own tables. */ + /* pledgedSrcSize == 0 means 0! */ + + if (cdict->matchState.dedicatedDictSearch) { + ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); + } + + params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, + cdict->dictContentSize, ZSTD_cpm_attachDict, + params.useRowMatchFinder); + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_makeClean, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); + } + + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; + if (cdictLen == 0) { + /* don't even attach dictionaries with no contents */ + DEBUGLOG(4, "skipping attaching empty dictionary"); + } else { + DEBUGLOG(4, "attaching dictionary into context"); + cctx->blockState.matchState.dictMatchState = &cdict->matchState; + + /* prep working match state so dict matches never have negative indices + * when they are translated to the working context's index space. */ + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { + cctx->blockState.matchState.window.nextSrc = + cctx->blockState.matchState.window.base + cdictEnd; + ZSTD_window_clear(&cctx->blockState.matchState.window); + } + /* loadedDictEnd is expressed within the referential of the active context */ + cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; + } } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, + ZSTD_compressionParameters const* cParams) { + if (ZSTD_CDictIndicesAreTagged(cParams)){ + /* Remove tags from the CDict table if they are present. + * See docs on "short cache" in zstd_compress_internal.h for context. */ + size_t i; + for (i = 0; i < tableSize; i++) { + U32 const taggedIndex = src[i]; + U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS; + dst[i] = index; + } + } else { + ZSTD_memcpy(dst, src, tableSize * sizeof(U32)); + } +} + +static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + + assert(!cdict->matchState.dedicatedDictSearch); + DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + + { unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Copy only compression parameters related to tables. */ + params.cParams = *cdict_cParams; + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); + assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); + } + + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + assert(params.useRowMatchFinder != ZSTD_ps_auto); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) + ? ((size_t)1 << cdict_cParams->chainLog) + : 0; + size_t const hSize = (size_t)1 << cdict_cParams->hashLog; + + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize, cdict_cParams); + + /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ + if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize, cdict_cParams); + } + /* copy tag table */ + if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { + size_t const tagTableSize = hSize; + ZSTD_memcpy(cctx->blockState.matchState.tagTable, + cdict->matchState.tagTable, + tagTableSize); + cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; + } + } + + /* Zero the hashTable3, since the cdict never fills it */ + { int const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + assert(cdict->matchState.hashLog3 == 0); + ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + + /* copy dictionary offsets */ + { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; + ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +/* We have a choice between copying the dictionary context into the working + * context, or referencing the dictionary context from the working context + * in-place. We decide here which strategy to use. */ +static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + + DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", + (unsigned)pledgedSrcSize); + + if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { + return ZSTD_resetCCtx_byAttachingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } else { + return ZSTD_resetCCtx_byCopyingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } +} + +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * The "context", in this case, refers to the hash and chain tables, + * entropy tables, and dictionary references. + * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. + * @return : 0, or an error code */ +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, + const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, + "Can't copy a ctx that's not in init stage."); + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); + ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + { ZSTD_CCtx_params params = dstCCtx->requestedParams; + /* Copy only compression parameters related to tables. */ + params.cParams = srcCCtx->appliedParams.cParams; + assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto); + params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; + params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter; + params.ldmParams = srcCCtx->appliedParams.ldmParams; + params.fParams = fParams; + params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; + ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff); + assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); + assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); + assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); + assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); + assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); + } + + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, + srcCCtx->appliedParams.useRowMatchFinder, + 0 /* forDDSDict */) + ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) + : 0; + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; + int const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + + /* copy dictionary offsets */ + { + const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; + ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + dstCCtx->dictID = srcCCtx->dictID; + dstCCtx->dictContentSize = srcCCtx->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); + + return 0; +} + +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); + + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, + fParams, pledgedSrcSize, + zbuff); +} + + +#define ZSTD_ROWSIZE 16 +/*! ZSTD_reduceTable() : + * reduce table indexes by `reducerValue`, or squash to zero. + * PreserveMark preserves "unsorted mark" for btlazy2 strategy. + * It must be set to a clear 0/1 value, to remove branch during inlining. + * Presume table size is a multiple of ZSTD_ROWSIZE + * to help auto-vectorization */ +FORCE_INLINE_TEMPLATE void +ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) +{ + int const nbRows = (int)size / ZSTD_ROWSIZE; + int cellNb = 0; + int rowNb; + /* Protect special index values < ZSTD_WINDOW_START_INDEX. */ + U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX; + assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ + assert(size < (1U<<31)); /* can be casted to int */ + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * + * This function however is intended to operate on those dirty tables and + * re-clean them. So when this function is used correctly, we can unpoison + * the memory it operated on. This introduces a blind spot though, since + * if we now try to operate on __actually__ poisoned memory, we will not + * detect that. */ + __msan_unpoison(table, size * sizeof(U32)); +#endif + + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { + int column; + for (column=0; columncParams.hashLog; + ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); + } + + if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) + ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); + else + ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + } + + if (ms->hashLog3) { + U32 const h3Size = (U32)1 << ms->hashLog3; + ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); + } +} + + +/*-******************************************************* +* Block entropic compression +*********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + int longOffsets = 0; + assert(nbSeq <= seqStorePtr->maxNbSeq); + for (u=0; u= STREAM_ACCUMULATOR_MIN)); + if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) + longOffsets = 1; + } + if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + return longOffsets; +} + +/* ZSTD_useTargetCBlockSize(): + * Returns if target compressed block size param is being used. + * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); + return (cctxParams->targetCBlockSize != 0); +} + +/* ZSTD_blockSplitterEnabled(): + * Returns if block splitting param is being used + * If used, compression will do best effort to split a block in order to improve compression ratio. + * At the time this function is called, the parameter must be finalized. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter); + assert(cctxParams->useBlockSplitter != ZSTD_ps_auto); + return (cctxParams->useBlockSplitter == ZSTD_ps_enable); +} + +/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types + * and size of the sequences statistics + */ +typedef struct { + U32 LLtype; + U32 Offtype; + U32 MLtype; + size_t size; + size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ + int longOffsets; +} ZSTD_symbolEncodingTypeStats_t; + +/* ZSTD_buildSequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field. + * Modifies `nextEntropy` to have the appropriate values as a side effect. + * nbSeq must be greater than 0. + * + * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildSequencesStatistics( + const seqStore_t* seqStorePtr, size_t nbSeq, + const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, + BYTE* dst, const BYTE* const dstEnd, + ZSTD_strategy strategy, unsigned* countWorkspace, + void* entropyWorkspace, size_t entropyWkspSize) +{ + BYTE* const ostart = dst; + const BYTE* const oend = dstEnd; + BYTE* op = ostart; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + ZSTD_symbolEncodingTypeStats_t stats; + + stats.lastCountSize = 0; + /* convert length/distances into codes */ + stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); + assert(op <= oend); + assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ + /* build CTable for Literal Lengths */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype, + countWorkspace, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->litlengthCTable, + sizeof(prevEntropy->litlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); + stats.size = countSize; + return stats; + } + if (stats.LLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for Offsets */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype, + countWorkspace, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->offcodeCTable, + sizeof(prevEntropy->offcodeCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); + stats.size = countSize; + return stats; + } + if (stats.Offtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for MatchLengths */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype, + countWorkspace, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->matchlengthCTable, + sizeof(prevEntropy->matchlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); + stats.size = countSize; + return stats; + } + if (stats.MLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + stats.size = (size_t)(op-ostart); + return stats; +} + +/* ZSTD_entropyCompressSeqStore_internal(): + * compresses both literals and sequences + * Returns compressed size of block, or a zstd error. + */ +#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore_internal( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + void* entropyWorkspace, size_t entropyWkspSize, + const int bmi2) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + unsigned* count = (unsigned*)entropyWorkspace; + FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; + const seqDef* const sequences = seqStorePtr->sequencesStart; + const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t lastCountSize; + int longOffsets = 0; + + entropyWorkspace = count + (MaxSeq + 1); + entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); + + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity); + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); + + /* Compress literals */ + { const BYTE* const literals = seqStorePtr->litStart; + size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + /* Base suspicion of uncompressibility on ratio of literals to sequences */ + unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); + size_t const litSize = (size_t)(seqStorePtr->lit - literals); + + size_t const cSize = ZSTD_compressLiterals( + op, dstCapacity, + literals, litSize, + entropyWorkspace, entropyWkspSize, + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + suspectUncompressible, bmi2); + FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); + assert(cSize <= dstCapacity); + op += cSize; + } + + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, "Can't fit seq hdr in output buf!"); + if (nbSeq < 128) { + *op++ = (BYTE)nbSeq; + } else if (nbSeq < LONGNBSEQ) { + op[0] = (BYTE)((nbSeq>>8) + 0x80); + op[1] = (BYTE)nbSeq; + op+=2; + } else { + op[0]=0xFF; + MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); + op+=3; + } + assert(op <= oend); + if (nbSeq==0) { + /* Copy the old tables over as if we repeated them */ + ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); + return (size_t)(op - ostart); + } + { BYTE* const seqHead = op++; + /* build stats for sequences */ + const ZSTD_symbolEncodingTypeStats_t stats = + ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + &prevEntropy->fse, &nextEntropy->fse, + op, oend, + strategy, count, + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); + lastCountSize = stats.lastCountSize; + op += stats.size; + longOffsets = stats.longOffsets; + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, (size_t)(oend - op), + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + assert(op <= oend); + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ + if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { + /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } + } + + DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); + return (size_t)(op - ostart); +} + +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) +{ + size_t const cSize = ZSTD_entropyCompressSeqStore_internal( + seqStorePtr, prevEntropy, nextEntropy, cctxParams, + dst, dstCapacity, + entropyWorkspace, entropyWkspSize, bmi2); + if (cSize == 0) return 0; + /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. + * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. + */ + if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) { + DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity); + return 0; /* block not compressed */ + } + FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); + + /* Check compressibility */ + { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); + if (cSize >= maxCSize) return 0; /* block not compressed */ + } + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); + /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly. + * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above. + */ + assert(cSize < ZSTD_BLOCKSIZE_MAX); + return cSize; +} + +/* ZSTD_selectBlockCompressor() : + * Not static, but internal use only (used by long distance matcher) + * assumption : strat is a valid strategy */ +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) +{ + static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { + { ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, + ZSTD_compressBlock_doubleFast, + ZSTD_compressBlock_greedy, + ZSTD_compressBlock_lazy, + ZSTD_compressBlock_lazy2, + ZSTD_compressBlock_btlazy2, + ZSTD_compressBlock_btopt, + ZSTD_compressBlock_btultra, + ZSTD_compressBlock_btultra2 }, + { ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, + ZSTD_compressBlock_doubleFast_extDict, + ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_lazy_extDict, + ZSTD_compressBlock_lazy2_extDict, + ZSTD_compressBlock_btlazy2_extDict, + ZSTD_compressBlock_btopt_extDict, + ZSTD_compressBlock_btultra_extDict, + ZSTD_compressBlock_btultra_extDict }, + { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, + ZSTD_compressBlock_fast_dictMatchState, + ZSTD_compressBlock_doubleFast_dictMatchState, + ZSTD_compressBlock_greedy_dictMatchState, + ZSTD_compressBlock_lazy_dictMatchState, + ZSTD_compressBlock_lazy2_dictMatchState, + ZSTD_compressBlock_btlazy2_dictMatchState, + ZSTD_compressBlock_btopt_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState }, + { NULL /* default for 0 */, + NULL, + NULL, + ZSTD_compressBlock_greedy_dedicatedDictSearch, + ZSTD_compressBlock_lazy_dedicatedDictSearch, + ZSTD_compressBlock_lazy2_dedicatedDictSearch, + NULL, + NULL, + NULL, + NULL } + }; + ZSTD_blockCompressor selectedCompressor; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); + if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { + static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = { + { ZSTD_compressBlock_greedy_row, + ZSTD_compressBlock_lazy_row, + ZSTD_compressBlock_lazy2_row }, + { ZSTD_compressBlock_greedy_extDict_row, + ZSTD_compressBlock_lazy_extDict_row, + ZSTD_compressBlock_lazy2_extDict_row }, + { ZSTD_compressBlock_greedy_dictMatchState_row, + ZSTD_compressBlock_lazy_dictMatchState_row, + ZSTD_compressBlock_lazy2_dictMatchState_row }, + { ZSTD_compressBlock_greedy_dedicatedDictSearch_row, + ZSTD_compressBlock_lazy_dedicatedDictSearch_row, + ZSTD_compressBlock_lazy2_dedicatedDictSearch_row } + }; + DEBUGLOG(4, "Selecting a row-based matchfinder"); + assert(useRowMatchFinder != ZSTD_ps_auto); + selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; + } else { + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; + } + assert(selectedCompressor != NULL); + return selectedCompressor; +} + +static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, + const BYTE* anchor, size_t lastLLSize) +{ + ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; +} + +void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthType = ZSTD_llt_none; +} + +/* ZSTD_postProcessSequenceProducerResult() : + * Validates and post-processes sequences obtained through the external matchfinder API: + * - Checks whether nbExternalSeqs represents an error condition. + * - Appends a block delimiter to outSeqs if one is not already present. + * See zstd.h for context regarding block delimiters. + * Returns the number of sequences after post-processing, or an error code. */ +static size_t ZSTD_postProcessSequenceProducerResult( + ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize +) { + RETURN_ERROR_IF( + nbExternalSeqs > outSeqsCapacity, + sequenceProducer_failed, + "External sequence producer returned error code %lu", + (unsigned long)nbExternalSeqs + ); + + RETURN_ERROR_IF( + nbExternalSeqs == 0 && srcSize > 0, + sequenceProducer_failed, + "Got zero sequences from external sequence producer for a non-empty src buffer!" + ); + + if (srcSize == 0) { + ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence)); + return 1; + } + + { + ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1]; + + /* We can return early if lastSeq is already a block delimiter. */ + if (lastSeq.offset == 0 && lastSeq.matchLength == 0) { + return nbExternalSeqs; + } + + /* This error condition is only possible if the external matchfinder + * produced an invalid parse, by definition of ZSTD_sequenceBound(). */ + RETURN_ERROR_IF( + nbExternalSeqs == outSeqsCapacity, + sequenceProducer_failed, + "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!" + ); + + /* lastSeq is not a block delimiter, so we need to append one. */ + ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence)); + return nbExternalSeqs + 1; + } +} + +/* ZSTD_fastSequenceLengthSum() : + * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*. + * Similar to another function in zstd_compress.c (determine_blockSize), + * except it doesn't check for a block delimiter to end summation. + * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P). + * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */ +static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) { + size_t matchLenSum, litLenSum, i; + matchLenSum = 0; + litLenSum = 0; + for (i = 0; i < seqBufSize; i++) { + litLenSum += seqBuf[i].litLength; + matchLenSum += seqBuf[i].matchLength; + } + return litLenSum + matchLenSum; +} + +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + ZSTD_matchState_t* const ms = &zc->blockState.matchState; + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + /* Assert that we have correctly flushed the ctx params into the ms's copy */ + ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); + } else { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + } + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ + } + ZSTD_resetSeqStore(&(zc->seqStore)); + /* required for optimal parser to read stats from dictionary */ + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + /* tell the optimal parser how we expect to compress literals */ + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; + /* a gap between an attached dict and the current window is not safe, + * they must remain adjacent, + * and when that stops being the case, the dict must be unset */ + assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); + + /* limited update after a very long match */ + { const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 curr = (U32)(istart-base); + if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ + if (curr > ms->nextToUpdate + 384) + ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); + } + + /* select and store sequences */ + { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + size_t lastLLSize; + { int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; + } + if (zc->externSeqStore.pos < zc->externSeqStore.size) { + assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + zc->appliedParams.useSequenceProducer, + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&zc->externSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(zc->externSeqStore.pos <= zc->externSeqStore.size); + } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + rawSeqStore_t ldmSeqStore = kNullRawSeqStore; + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + zc->appliedParams.useSequenceProducer, + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + + ldmSeqStore.seq = zc->ldmSequences; + ldmSeqStore.capacity = zc->maxNbLdmSequences; + /* Updates ldmSeqStore.size */ + FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, + &zc->appliedParams.ldmParams, + src, srcSize), ""); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&ldmSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(ldmSeqStore.pos == ldmSeqStore.size); + } else if (zc->appliedParams.useSequenceProducer) { + assert( + zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize) + ); + assert(zc->externalMatchCtx.mFinder != NULL); + + { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; + + size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)( + zc->externalMatchCtx.mState, + zc->externalMatchCtx.seqBuffer, + zc->externalMatchCtx.seqBufferCapacity, + src, srcSize, + NULL, 0, /* dict and dictSize, currently not supported */ + zc->appliedParams.compressionLevel, + windowSize + ); + + size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( + zc->externalMatchCtx.seqBuffer, + nbExternalSeqs, + zc->externalMatchCtx.seqBufferCapacity, + srcSize + ); + + /* Return early if there is no error, since we don't need to worry about last literals */ + if (!ZSTD_isError(nbPostProcessedSeqs)) { + ZSTD_sequencePosition seqPos = {0,0,0}; + size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs); + RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!"); + FORWARD_IF_ERROR( + ZSTD_copySequencesToSeqStoreExplicitBlockDelim( + zc, &seqPos, + zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs, + src, srcSize, + zc->appliedParams.searchForExternalRepcodes + ), + "Failed to copy external sequences to seqStore!" + ); + ms->ldmSeqStore = NULL; + DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs); + return ZSTDbss_compress; + } + + /* Propagate the error if fallback is disabled */ + if (!zc->appliedParams.enableMatchFinderFallback) { + return nbPostProcessedSeqs; + } + + /* Fallback to software matchfinder */ + { ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + DEBUGLOG( + 5, + "External sequence producer returned error code %lu. Falling back to internal parser.", + (unsigned long)nbExternalSeqs + ); + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } } + } else { /* not long range mode and no external matchfinder */ + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } + { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); + } } + return ZSTDbss_compress; +} + +static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +{ + const seqStore_t* seqStore = ZSTD_getSeqStore(zc); + const seqDef* seqStoreSeqs = seqStore->sequencesStart; + size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs; + size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart); + size_t literalsRead = 0; + size_t lastLLSize; + + ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t i; + repcodes_t updatedRepcodes; + + assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); + /* Ensure we have enough space for last literals "sequence" */ + assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); + ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (i = 0; i < seqStoreSeqSize; ++i) { + U32 rawOffset = seqStoreSeqs[i].offBase - ZSTD_REP_NUM; + outSeqs[i].litLength = seqStoreSeqs[i].litLength; + outSeqs[i].matchLength = seqStoreSeqs[i].mlBase + MINMATCH; + outSeqs[i].rep = 0; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (seqStoreSeqs[i].offBase <= ZSTD_REP_NUM) { + /* Derive the correct offset corresponding to a repcode */ + outSeqs[i].rep = seqStoreSeqs[i].offBase; + if (outSeqs[i].litLength != 0) { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1]; + } else { + if (outSeqs[i].rep == 3) { + rawOffset = updatedRepcodes.rep[0] - 1; + } else { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep]; + } + } + } + outSeqs[i].offset = rawOffset; + /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode + so we provide seqStoreSeqs[i].offset - 1 */ + ZSTD_updateRep(updatedRepcodes.rep, + seqStoreSeqs[i].offBase, + seqStoreSeqs[i].litLength == 0); + literalsRead += outSeqs[i].litLength; + } + /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. + * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker + * for the block boundary, according to the API. + */ + assert(seqStoreLiteralsSize >= literalsRead); + lastLLSize = seqStoreLiteralsSize - literalsRead; + outSeqs[i].litLength = (U32)lastLLSize; + outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; + seqStoreSeqSize++; + zc->seqCollector.seqIndex += seqStoreSeqSize; +} + +size_t ZSTD_sequenceBound(size_t srcSize) { + return (srcSize / ZSTD_MINMATCH_MIN) + 1; +} + +size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; + + RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); + + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_customFree(dst, ZSTD_defaultCMem); + return zc->seqCollector.seqIndex; +} + +size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { + size_t in = 0; + size_t out = 0; + for (; in < seqsSize; ++in) { + if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { + if (in != seqsSize - 1) { + sequences[in+1].litLength += sequences[in].litLength; + } + } else { + sequences[out] = sequences[in]; + ++out; + } + } + return out; +} + +/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ +static int ZSTD_isRLE(const BYTE* src, size_t length) { + const BYTE* ip = src; + const BYTE value = ip[0]; + const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); + const size_t unrollSize = sizeof(size_t) * 4; + const size_t unrollMask = unrollSize - 1; + const size_t prefixLength = length & unrollMask; + size_t i; + if (length == 1) return 1; + /* Check if prefix is RLE first before using unrolled loop */ + if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { + return 0; + } + for (i = prefixLength; i != length; i += unrollSize) { + size_t u; + for (u = 0; u < unrollSize; u += sizeof(size_t)) { + if (MEM_readST(ip + i + u) != valueST) { + return 0; + } } } + return 1; +} + +/* Returns true if the given block may be RLE. + * This is just a heuristic based on the compressibility. + * It may return both false positives and false negatives. + */ +static int ZSTD_maybeRLE(seqStore_t const* seqStore) +{ + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); + + return nbSeqs < 4 && nbLits < 10; +} + +static void +ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) +{ + ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; + bs->prevCBlock = bs->nextCBlock; + bs->nextCBlock = tmp; +} + +/* Writes the block header */ +static void +writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) +{ + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); +} + +/** ZSTD_buildBlockEntropyStats_literals() : + * Builds entropy for the literals. + * Stores literals block type (raw, rle, compressed, repeat) and + * huffman description table to hufMetadata. + * Requires ENTROPY_WORKSPACE_SIZE workspace + * @return : size of huffman description table, or an error code + */ +static size_t +ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int literalsCompressionIsDisabled, + void* workspace, size_t wkspSize, + int hufFlags) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); + BYTE* const nodeWksp = countWkspStart + countWkspSize; + const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + unsigned huffLog = LitHufLog; + HUF_repeat repeat = prevHuf->repeatMode; + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (literalsCompressionIsDisabled) { + DEBUGLOG(5, "set_basic - disabled"); + hufMetadata->hType = set_basic; + return 0; + } + + /* small ? don't even attempt compression (speed opt) */ +#ifndef COMPRESS_LITERALS_SIZE_MIN +# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */ +#endif + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) { + DEBUGLOG(5, "set_basic - too small"); + hufMetadata->hType = set_basic; + return 0; + } } + + /* Scan input and build symbol stats */ + { size_t const largest = + HIST_count_wksp (countWksp, &maxSymbolValue, + (const BYTE*)src, srcSize, + workspace, wkspSize); + FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); + if (largest == srcSize) { + /* only one literal symbol */ + DEBUGLOG(5, "set_rle"); + hufMetadata->hType = set_rle; + return 0; + } + if (largest <= (srcSize >> 7)+4) { + /* heuristic: likely not compressible */ + DEBUGLOG(5, "set_basic - no gain"); + hufMetadata->hType = set_basic; + return 0; + } } + + /* Validate the previous Huffman table */ + if (repeat == HUF_repeat_check + && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + repeat = HUF_repeat_none; + } + + /* Build Huffman Tree */ + ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags); + assert(huffLog <= LitHufLog); + { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, + maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); + huffLog = (U32)maxBits; + } + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable_wksp( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_repeat; + return 0; + } } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; + } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; + } +} + + +/* ZSTD_buildDummySequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic, + * and updates nextEntropy to the appropriate repeatMode. + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) +{ + ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; + nextEntropy->litlength_repeatMode = FSE_repeat_none; + nextEntropy->offcode_repeatMode = FSE_repeat_none; + nextEntropy->matchlength_repeatMode = FSE_repeat_none; + return stats; +} + +/** ZSTD_buildBlockEntropyStats_sequences() : + * Builds entropy for the sequences. + * Stores symbol compression modes and fse table to fseMetadata. + * Requires ENTROPY_WORKSPACE_SIZE wksp. + * @return : size of fse tables or error code */ +static size_t +ZSTD_buildBlockEntropyStats_sequences( + const seqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + BYTE* const ostart = fseMetadata->fseTablesBuffer; + BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); + BYTE* op = ostart; + unsigned* countWorkspace = (unsigned*)workspace; + unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); + size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); + ZSTD_symbolEncodingTypeStats_t stats; + + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); + stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + prevEntropy, nextEntropy, op, oend, + strategy, countWorkspace, + entropyWorkspace, entropyWorkspaceSize) + : ZSTD_buildDummySequencesStatistics(nextEntropy); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + fseMetadata->llType = (symbolEncodingType_e) stats.LLtype; + fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype; + fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype; + fseMetadata->lastCountSize = stats.lastCountSize; + return stats.size; +} + + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * Requires workspace size ENTROPY_WORKSPACE_SIZE + * @return : 0 on success, or an error code + * Note : also employed in superblock + */ +size_t ZSTD_buildBlockEntropyStats( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) +{ + size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); + int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0; + + entropyMetadata->hufMetadata.hufDesSize = + ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, + &prevEntropy->huf, &nextEntropy->huf, + &entropyMetadata->hufMetadata, + ZSTD_literalsCompressionIsDisabled(cctxParams), + workspace, wkspSize, hufFlags); + + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); + entropyMetadata->fseMetadata.fseTablesSize = + ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, + &prevEntropy->fse, &nextEntropy->fse, + cctxParams, + &entropyMetadata->fseMetadata, + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); + return 0; +} + +/* Returns the size estimate for the literals section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB); + U32 singleStream = litSize < 256; + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ +static size_t +ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, size_t nbSeq, unsigned maxCode, + const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + (void)defaultMax; + cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) { + return nbSeq * 10; + } + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits >> 3; +} + +/* Returns the size estimate for the sequences section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); + size_t cSeqSizeEstimate = 0; + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, + fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, + fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, + fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +/* Returns the size estimate for a given stream of literals, of, ll, ml */ +static size_t +ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) +{ + size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return seqSize + literalsSize + ZSTD_blockHeaderSize; +} + +/* Builds entropy statistics and uses them for blocksize estimation. + * + * @return: estimated compressed size of the seqStore, or a zstd error. + */ +static size_t +ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) +{ + ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; + DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), ""); + return ZSTD_estimateBlockSize( + seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), + seqStore->ofCode, seqStore->llCode, seqStore->mlCode, + (size_t)(seqStore->sequences - seqStore->sequencesStart), + &zc->blockState.nextCBlock->entropy, + entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, + (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); +} + +/* Returns literals bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) +{ + size_t literalsBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + seqDef const seq = seqStore->sequencesStart[i]; + literalsBytes += seq.litLength; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { + literalsBytes += 0x10000; + } } + return literalsBytes; +} + +/* Returns match bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) +{ + size_t matchBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + seqDef seq = seqStore->sequencesStart[i]; + matchBytes += seq.mlBase + MINMATCH; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { + matchBytes += 0x10000; + } } + return matchBytes; +} + +/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). + * Stores the result in resultSeqStore. + */ +static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, + const seqStore_t* originalSeqStore, + size_t startIdx, size_t endIdx) +{ + *resultSeqStore = *originalSeqStore; + if (startIdx > 0) { + resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + } + + /* Move longLengthPos into the correct position if necessary */ + if (originalSeqStore->longLengthType != ZSTD_llt_none) { + if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { + resultSeqStore->longLengthType = ZSTD_llt_none; + } else { + resultSeqStore->longLengthPos -= (U32)startIdx; + } + } + resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; + if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { + /* This accounts for possible last literals if the derived chunk reaches the end of the block */ + assert(resultSeqStore->lit == originalSeqStore->lit); + } else { + size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + resultSeqStore->lit = resultSeqStore->litStart + literalsBytes; + } + resultSeqStore->llCode += startIdx; + resultSeqStore->mlCode += startIdx; + resultSeqStore->ofCode += startIdx; +} + +/** + * Returns the raw offset represented by the combination of offBase, ll0, and repcode history. + * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq(). + */ +static U32 +ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0) +{ + U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */ + assert(OFFBASE_IS_REPCODE(offBase)); + if (adjustedRepCode == ZSTD_REP_NUM) { + assert(ll0); + /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 + * This is only valid if it results in a valid offset value, aka > 0. + * Note : it may happen that `rep[0]==1` in exceptional circumstances. + * In which case this function will return 0, which is an invalid offset. + * It's not an issue though, since this value will be + * compared and discarded within ZSTD_seqStore_resolveOffCodes(). + */ + return rep[0] - 1; + } + return rep[adjustedRepCode]; +} + +/** + * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise + * due to emission of RLE/raw blocks that disturb the offset history, + * and replaces any repcodes within the seqStore that may be invalid. + * + * dRepcodes are updated as would be on the decompression side. + * cRepcodes are updated exactly in accordance with the seqStore. + * + * Note : this function assumes seq->offBase respects the following numbering scheme : + * 0 : invalid + * 1-3 : repcode 1-3 + * 4+ : real_offset+3 + */ +static void +ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, + const seqStore_t* const seqStore, U32 const nbSeq) +{ + U32 idx = 0; + U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq; + for (; idx < nbSeq; ++idx) { + seqDef* const seq = seqStore->sequencesStart + idx; + U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx); + U32 const offBase = seq->offBase; + assert(offBase > 0); + if (OFFBASE_IS_REPCODE(offBase)) { + U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0); + U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0); + /* Adjust simulated decompression repcode history if we come across a mismatch. Replace + * the repcode with the offset it actually references, determined by the compression + * repcode history. + */ + if (dRawOffset != cRawOffset) { + seq->offBase = OFFSET_TO_OFFBASE(cRawOffset); + } + } + /* Compression repcode history is always updated with values directly from the unmodified seqStore. + * Decompression repcode history may use modified seq->offset value taken from compression repcode history. + */ + ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0); + ZSTD_updateRep(cRepcodes->rep, offBase, ll0); + } +} + +/* ZSTD_compressSeqStore_singleBlock(): + * Compresses a seqStore into a block with a block header, into the buffer dst. + * + * Returns the total size of that block (including header) or a ZSTD error code. + */ +static size_t +ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, + const seqStore_t* const seqStore, + repcodes_t* const dRep, repcodes_t* const cRep, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock, U32 isPartition) +{ + const U32 rleMaxLength = 25; + BYTE* op = (BYTE*)dst; + const BYTE* ip = (const BYTE*)src; + size_t cSize; + size_t cSeqsSize; + + /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */ + repcodes_t const dRepOriginal = *dRep; + DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock"); + if (isPartition) + ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart)); + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit"); + cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); + + if (!zc->isFirstBlock && + cSeqsSize < rleMaxLength && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + cSeqsSize = 1; + } + + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + if (cSeqsSize == 0) { + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "Nocompress block failed"); + DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else if (cSeqsSize == 1) { + cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "RLE compress block failed"); + DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + writeBlockHeader(op, cSeqsSize, srcSize, lastBlock); + cSize = ZSTD_blockHeaderSize + cSeqsSize; + DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize); + } + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +/* Struct to keep track of where we are in our recursive calls. */ +typedef struct { + U32* splitLocations; /* Array of split indices */ + size_t idx; /* The current index within splitLocations being worked on */ +} seqStoreSplits; + +#define MIN_SEQUENCES_BLOCK_SPLITTING 300 + +/* Helper function to perform the recursive search for block splits. + * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. + * If advantageous to split, then we recurse down the two sub-blocks. + * If not, or if an error occurred in estimation, then we do not recurse. + * + * Note: The recursion depth is capped by a heuristic minimum number of sequences, + * defined by MIN_SEQUENCES_BLOCK_SPLITTING. + * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). + * In practice, recursion depth usually doesn't go beyond 4. + * + * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. + * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize + * maximum of 128 KB, this value is actually impossible to reach. + */ +static void +ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, + ZSTD_CCtx* zc, const seqStore_t* origSeqStore) +{ + seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; + seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; + seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; + size_t estimatedOriginalSize; + size_t estimatedFirstHalfSize; + size_t estimatedSecondHalfSize; + size_t midIdx = (startIdx + endIdx)/2; + + DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); + assert(endIdx >= startIdx); + if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { + DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); + return; + } + ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); + ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); + ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); + estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc); + estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc); + estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc); + DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", + estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); + if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { + return; + } + if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { + DEBUGLOG(5, "split decided at seqNb:%zu", midIdx); + ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); + splits->splitLocations[splits->idx] = (U32)midIdx; + splits->idx++; + ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); + } +} + +/* Base recursive function. + * Populates a table with intra-block partition indices that can improve compression ratio. + * + * @return: number of splits made (which equals the size of the partition table - 1). + */ +static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) +{ + seqStoreSplits splits; + splits.splitLocations = partitions; + splits.idx = 0; + if (nbSeq <= 4) { + DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq); + /* Refuse to try and split anything with less than 4 sequences */ + return 0; + } + ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); + splits.splitLocations[splits.idx] = nbSeq; + DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1); + return splits.idx; +} + +/* ZSTD_compressBlock_splitBlock(): + * Attempts to split a given block into multiple blocks to improve compression ratio. + * + * Returns combined size of all blocks (which includes headers), or a ZSTD error code. + */ +static size_t +ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t blockSize, + U32 lastBlock, U32 nbSeq) +{ + size_t cSize = 0; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + size_t i = 0; + size_t srcBytesTotal = 0; + U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ + seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; + seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; + size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + + /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history + * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two + * separate repcode histories that simulate repcode history on compression and decompression side, + * and use the histories to determine whether we must replace a particular repcode with its raw offset. + * + * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed + * or RLE. This allows us to retrieve the offset value that an invalid repcode references within + * a nocompress/RLE block. + * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use + * the replacement offset value rather than the original repcode to update the repcode history. + * dRep also will be the final repcode history sent to the next block. + * + * See ZSTD_seqStore_resolveOffCodes() for more details. + */ + repcodes_t dRep; + repcodes_t cRep; + ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t)); + + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + if (numSplits == 0) { + size_t cSizeSingleBlock = + ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, + &dRep, &cRep, + op, dstCapacity, + ip, blockSize, + lastBlock, 0 /* isPartition */); + FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); + assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX); + assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize); + return cSizeSingleBlock; + } + + ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); + for (i = 0; i <= numSplits; ++i) { + size_t cSizeChunk; + U32 const lastPartition = (i == numSplits); + U32 lastBlockEntireSrc = 0; + + size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); + srcBytesTotal += srcBytes; + if (lastPartition) { + /* This is the final partition, need to account for possible last literals */ + srcBytes += blockSize - srcBytesTotal; + lastBlockEntireSrc = lastBlock; + } else { + ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); + } + + cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore, + &dRep, &cRep, + op, dstCapacity, + ip, srcBytes, + lastBlockEntireSrc, 1 /* isPartition */); + DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size", + ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); + FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); + + ip += srcBytes; + op += cSizeChunk; + dstCapacity -= cSizeChunk; + cSize += cSizeChunk; + *currSeqStore = *nextSeqStore; + assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize); + } + /* cRep and dRep may have diverged during the compression. + * If so, we use the dRep repcodes for the next block. + */ + ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t)); + return cSize; +} + +static size_t +ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 lastBlock) +{ + U32 nbSeq; + size_t cSize; + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); + assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block"); + return cSize; + } + nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); + } + + cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); + FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); + return cSize; +} + +static size_t +ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 frame) +{ + /* This is an estimated upper bound for the length of an rle block. + * This isn't the actual upper bound. + * Finding the real threshold needs further investigation. + */ + const U32 rleMaxLength = 25; + size_t cSize; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + } + + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + /* encode sequences and literals */ + cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + dst, dstCapacity, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + + if (frame && + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + cSize < rleMaxLength && + ZSTD_isRLE(ip, srcSize)) + { + cSize = 1; + op[0] = ip[0]; + } + +out: + if (!ZSTD_isError(cSize) && cSize > 1) { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + } + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const size_t bss, U32 lastBlock) +{ + DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); + if (bss == ZSTDbss_compress) { + if (/* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + ZSTD_maybeRLE(&zc->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) + { + return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); + } + /* Attempt superblock compression. + * + * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the + * standard ZSTD_compressBound(). This is a problem, because even if we have + * space now, taking an extra byte now could cause us to run out of space later + * and violate ZSTD_compressBound(). + * + * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize. + * + * In order to respect ZSTD_compressBound() we must attempt to emit a raw + * uncompressed block in these cases: + * * cSize == 0: Return code for an uncompressed block. + * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize). + * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of + * output space. + * * cSize >= blockBound(srcSize): We have expanded the block too much so + * emit an uncompressed block. + */ + { size_t const cSize = + ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + if (cSize != ERROR(dstSize_tooSmall)) { + size_t const maxCSize = + srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); + if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return cSize; + } + } + } + } /* if (bss == ZSTDbss_compress)*/ + + DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); + /* Superblock compression failed, attempt to emit a single no compress block. + * The decoder will be able to stream this block since it is uncompressed. + */ + return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); +} + +static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock) +{ + size_t cSize = 0; + const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + + cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + void const* ip, + void const* iend) +{ + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const maxDist = (U32)1 << params->cParams.windowLog; + if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) { + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_cwksp_mark_tables_dirty(ws); + ZSTD_reduceIndex(ms, params, correction); + ZSTD_cwksp_mark_tables_clean(ws); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + /* invalidate dictionaries on overflow correction */ + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } +} + +/*! ZSTD_compress_frameChunk() : +* Compress a chunk of data into one or multiple blocks. +* All blocks will be terminated, all input will be consumed. +* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. +* Frame is supposed already started (header already produced) +* @return : compressed size, or an error code +*/ +static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); + + DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + if (cctx->appliedParams.fParams.checksumFlag && srcSize) + XXH64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1, + dstSize_tooSmall, + "not enough space to store compressed block"); + if (remaining < blockSize) blockSize = remaining; + + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + + /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ + if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; + + { size_t cSize; + if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); + assert(cSize > 0); + assert(cSize <= blockSize + ZSTD_blockHeaderSize); + } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed"); + assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); + } else { + cSize = ZSTD_compressBlock_internal(cctx, + op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, + ip, blockSize, 1 /* frame */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed"); + + if (cSize == 0) { /* block is not compressible */ + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + } else { + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + cSize += ZSTD_blockHeaderSize; + } + } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ + + + ip += blockSize; + assert(remaining >= blockSize); + remaining -= blockSize; + op += cSize; + assert(dstCapacity >= cSize); + dstCapacity -= cSize; + cctx->isFirstBlock = 0; + DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", + (unsigned)cSize); + } } + + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; + return (size_t)(op-ostart); +} + + +static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) +{ BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params->fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params->cParams.windowLog; + U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params->fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ + BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + size_t pos=0; + + assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, + "dst buf is too small to fit worst-case frame header size."); + DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", + !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); + if (params->format == ZSTD_f_zstd1) { + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + pos = 4; + } + op[pos++] = frameHeaderDescriptionByte; + if (!singleSegment) op[pos++] = windowLogByte; + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } + switch(fcsCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; + } + return pos; +} + +/* ZSTD_writeSkippableFrame_advanced() : + * Writes out a skippable frame with the specified magic number variant (16 are supported), + * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. + * + * Returns the total number of bytes written, or a ZSTD error code. + */ +size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant) { + BYTE* op = (BYTE*)dst; + RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, + dstSize_tooSmall, "Not enough room for skippable frame"); + RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); + RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); + + MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); + MEM_writeLE32(op+4, (U32)srcSize); + ZSTD_memcpy(op+8, src, srcSize); + return srcSize + ZSTD_SKIPPABLEHEADERSIZE; +} + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small (stage != ZSTDcs_init, stage_wrong, + "wrong cctx stage"); + RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable, + parameter_unsupported, + "incompatible with ldm"); + cctx->externSeqStore.seq = seq; + cctx->externSeqStore.size = nbSeq; + cctx->externSeqStore.capacity = nbSeq; + cctx->externSeqStore.pos = 0; + cctx->externSeqStore.posInSequence = 0; + return 0; +} + + +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, U32 lastFrameChunk) +{ + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + size_t fhSize = 0; + + DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", + cctx->stage, (unsigned)srcSize); + RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, + "missing init (ZSTD_compressBegin)"); + + if (frame && (cctx->stage==ZSTDcs_init)) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, + cctx->pledgedSrcSizePlusOne-1, cctx->dictID); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + assert(fhSize <= dstCapacity); + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (!srcSize) return fhSize; /* do not generate an empty block if no input */ + + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; + ms->nextToUpdate = ms->window.dictLimit; + } + if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); + } + + if (!frame) { + /* overflow check and correction for block mode */ + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, + src, (BYTE const*)src + srcSize); + } + + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); + { size_t const cSize = frame ? + ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); + FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed"); + cctx->consumedSrcSize += srcSize; + cctx->producedCSize += (cSize + fhSize); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + RETURN_ERROR_IF( + cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize >= %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + return cSize + fhSize; + } +} + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); +} + +/* NOTE: Must just wrap ZSTD_compressContinue_public() */ +size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize); +} + +static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx) +{ + ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; + assert(!ZSTD_checkCParams(cParams)); + return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); +} + +/* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */ +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + return ZSTD_getBlockSize_deprecated(cctx); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); + { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx); + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } + + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_loadDictionaryContent() : + * @return : 0, or an error code + */ +static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* src, size_t srcSize, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; + + /* Assert that the ms params match the params we're being given */ + ZSTD_assertEqualCParams(params->cParams, ms->cParams); + + { /* Ensure large dictionaries can't cause index overflow */ + + /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. + * Dictionaries right at the edge will immediately trigger overflow + * correction, but I don't want to insert extra constraints here. + */ + U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + + int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(¶ms->cParams); + if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) { + /* Some dictionary matchfinders in zstd use "short cache", + * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each + * CDict hashtable entry as a tag rather than as part of an index. + * When short cache is used, we need to truncate the dictionary + * so that its indices don't overlap with the tag. */ + U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); + assert(!loadLdmDict); + } + + /* If the dictionary is too large, only load the suffix of the dictionary. */ + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + if (srcSize > ZSTD_CHUNKSIZE_MAX) { + /* We must have cleared our windows when our source is this large. */ + assert(ZSTD_window_isEmpty(ms->window)); + if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window)); + } + ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); + + DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); + + if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */ + ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); + ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); + } + + /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */ + if (params->cParams.strategy < ZSTD_btultra) { + U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28); + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + ms->nextToUpdate = (U32)(ip - ms->window.base); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; + + if (srcSize <= HASH_READ_SIZE) return 0; + + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); + + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, dtlm, tfp); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + assert(srcSize >= HASH_READ_SIZE); + if (ms->dedicatedDictSearch) { + assert(ms->chainTable != NULL); + ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); + } else { + assert(params->useRowMatchFinder != ZSTD_ps_auto); + if (params->useRowMatchFinder == ZSTD_ps_enable) { + size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ZSTD_row_update(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using row-based hash table for lazy dict"); + } else { + ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using chain-based hash table for lazy dict"); + } + } + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + assert(srcSize >= HASH_READ_SIZE); + ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ms->nextToUpdate = (U32)(iend - ms->window.base); + return 0; +} + + +/* Dictionaries that assign zero probability to symbols that show up causes problems + * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check + * and only dictionaries with 100% valid symbols can be assumed valid. + */ +static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) +{ + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) { + return FSE_repeat_check; + } + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) { + return FSE_repeat_check; + } + } + return FSE_repeat_valid; +} + +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize) +{ + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ + const BYTE* const dictEnd = dictPtr + dictSize; + dictPtr += 8; + bs->entropy.huf.repeatMode = HUF_repeat_check; + + { unsigned maxSymbolValue = 255; + unsigned hasZeroWeights = 1; + size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, + dictEnd-dictPtr, &hasZeroWeights); + + /* We only set the loaded table as valid if it contains all non-zero + * weights. Otherwise, we set it to check */ + if (!hasZeroWeights) + bs->entropy.huf.repeatMode = HUF_repeat_valid; + + RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, ""); + dictPtr += hufHeaderSize; + } + + { unsigned offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + /* fill all offset symbols to avoid garbage at end of table */ + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.offcodeCTable, + offcodeNCount, MaxOff, offcodeLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.matchlengthCTable, + matchlengthNCount, matchlengthMaxValue, matchlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.litlengthCTable, + litlengthNCount, litlengthMaxValue, litlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + bs->rep[0] = MEM_readLE32(dictPtr+0); + bs->rep[1] = MEM_readLE32(dictPtr+4); + bs->rep[2] = MEM_readLE32(dictPtr+8); + dictPtr += 12; + + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ + } + /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ + bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); + + /* All repCodes must be <= dictContentSize and != 0 */ + { U32 u; + for (u=0; u<3; u++) { + RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); + RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); + } } } + + return dictPtr - (const BYTE*)dict; +} + +/* Dictionary format : + * See : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : dictID, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed >= 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* dict, size_t dictSize, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, + void* workspace) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + size_t dictID; + size_t eSize; + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); + assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); + + dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); + eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize); + FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); + dictPtr += eSize; + + { + size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); + } + return dictID; +} + +/** ZSTD_compress_insertDictionary() : +* @return : dictID, or an error code */ +static size_t +ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + const ZSTD_CCtx_params* params, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, + void* workspace) +{ + DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); + if ((dict==NULL) || (dictSize<8)) { + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + return 0; + } + + ZSTD_reset_compressedBlockState(bs); + + /* dict restricted modes */ + if (dictContentType == ZSTD_dct_rawContent) + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp); + + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_auto) { + DEBUGLOG(4, "raw content dictionary detected"); + return ZSTD_loadDictionaryContent( + ms, ls, ws, params, dict, dictSize, dtlm, tfp); + } + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + assert(0); /* impossible */ + } + + /* dict as full zstd dictionary */ + return ZSTD_loadZstdDictionary( + bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); +} + +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) + +/*! ZSTD_compressBegin_internal() : + * Assumption : either @dict OR @cdict (or none) is non-NULL, never both + * @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; +#if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +#endif + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if ( (cdict) + && (cdict->dictContentSize > 0) + && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { + return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); + } + + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + dictContentSize, + ZSTDcrp_makeClean, zbuff) , ""); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, + cdict->dictContentSize, cdict->dictContentType, dtlm, + ZSTD_tfp_forCCtx, cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= UINT_MAX); + cctx->dictID = (U32)dictID; + cctx->dictContentSize = dictContentSize; + } + return 0; +} + +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); + /* compression parameters verification and optimization */ + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); + return ZSTD_compressBegin_internal(cctx, + dict, dictSize, dictContentType, dtlm, + cdict, + params, pledgedSrcSize, + ZSTDb_not_buffered); +} + +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compressBegin_advanced_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, + NULL /*cdict*/, + &cctxParams, pledgedSrcSize); +} + +static size_t +ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_CCtx_params cctxParams; + { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); + } + DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); +} + +size_t +ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel); +} + +size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel); +} + + +/*! ZSTD_writeEpilogue() : +* Ends a frame. +* @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + size_t fhSize = 0; + + DEBUGLOG(4, "ZSTD_writeEpilogue"); + RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op-ostart; +} + +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) +{ +#if ZSTD_TRACE + if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) { + int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0; + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + trace.dictionaryID = cctx->dictID; + trace.dictionarySize = cctx->dictContentSize; + trace.uncompressedSize = cctx->consumedSrcSize; + trace.compressedSize = cctx->producedCSize + extraCSize; + trace.params = &cctx->appliedParams; + trace.cctx = cctx; + ZSTD_trace_compress_end(cctx->traceCtx, &trace); + } + cctx->traceCtx = 0; +#else + (void)cctx; + (void)extraCSize; +#endif +} + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, + dst, dstCapacity, src, srcSize, + 1 /* frame mode */, 1 /* last chunk */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed"); + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed"); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + DEBUGLOG(4, "end of frame : controlling src size"); + RETURN_ERROR_IF( + cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize = %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + ZSTD_CCtx_trace(cctx, endResult); + return cSize + endResult; +} + +/* NOTE: Must just wrap ZSTD_compressEnd_public() */ +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced"); + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compress_advanced_internal(cctx, + dst, dstCapacity, + src, srcSize, + dict, dictSize, + &cctx->simpleApiParams); +} + +/* Internal */ +size_t ZSTD_compress_advanced_internal( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + params, srcSize, ZSTDb_not_buffered) , ""); + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel) +{ + { + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); + assert(params.fParams.contentSizeFlag == 1); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + } + DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); +} + +size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); + assert(cctx != NULL); + return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + size_t result; +#if ZSTD_COMPRESS_HEAPMODE + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed"); + result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtx(cctx); +#else + ZSTD_CCtx ctxBody; + ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem); + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */ +#endif + return result; +} + + +/* ===== Dictionary API ===== */ + +/*! ZSTD_estimateCDictSize_advanced() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize_advanced( + size_t dictSize, ZSTD_compressionParameters cParams, + ZSTD_dictLoadMethod_e dictLoadMethod) +{ + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small + * in case we are using DDS with row-hash. */ + + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), + /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); +} + +size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); +} + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); +} + +static size_t ZSTD_initCDict_internal( + ZSTD_CDict* cdict, + const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_CCtx_params params) +{ + DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); + assert(!ZSTD_checkCParams(params.cParams)); + cdict->matchState.cParams = params.cParams; + cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { + cdict->dictContent = dictBuffer; + } else { + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); + RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); + cdict->dictContent = internalBuffer; + ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); + } + cdict->dictContentSize = dictSize; + cdict->dictContentType = dictContentType; + + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + + /* Reset the state to no dictionary */ + ZSTD_reset_compressedBlockState(&cdict->cBlockState); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + ¶ms.cParams, + params.useRowMatchFinder, + ZSTDcrp_makeClean, + ZSTDirp_reset, + ZSTD_resetTarget_CDict), ""); + /* (Maybe) load the dictionary + * Skips loading the dictionary if it is < 8 bytes. + */ + { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; + params.fParams.contentSizeFlag = 1; + { size_t const dictID = ZSTD_compress_insertDictionary( + &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, + ¶ms, cdict->dictContent, cdict->dictContentSize, + dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= (size_t)(U32)-1); + cdict->dictID = (U32)dictID; + } + } + + return 0; +} + +static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_compressionParameters cParams, + ZSTD_paramSwitch_e useRowMatchFinder, + U32 enableDedicatedDictSearch, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { size_t const workspaceSize = + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); + void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); + ZSTD_cwksp ws; + ZSTD_CDict* cdict; + + if (!workspace) { + ZSTD_customFree(workspace, customMem); + return NULL; + } + + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc); + + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + ZSTD_cwksp_move(&cdict->workspace, &ws); + cdict->customMem = customMem; + cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ + cdict->useRowMatchFinder = useRowMatchFinder; + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); + ZSTD_CCtxParams_init(&cctxParams, 0); + cctxParams.cParams = cParams; + cctxParams.customMem = customMem; + return ZSTD_createCDict_advanced2( + dictBuffer, dictSize, + dictLoadMethod, dictContentType, + &cctxParams, customMem); +} + +ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* originalCctxParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams = *originalCctxParams; + ZSTD_compressionParameters cParams; + ZSTD_CDict* cdict; + + DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + if (cctxParams.enableDedicatedDictSearch) { + cParams = ZSTD_dedicatedDictSearch_getCParams( + cctxParams.compressionLevel, dictSize); + ZSTD_overrideCParams(&cParams, &cctxParams.cParams); + } else { + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) { + /* Fall back to non-DDSS params */ + cctxParams.enableDedicatedDictSearch = 0; + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch); + cctxParams.cParams = cParams; + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + + cdict = ZSTD_createCDict_advanced_internal(dictSize, + dictLoadMethod, cctxParams.cParams, + cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, + customMem); + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + cctxParams) )) { + ZSTD_freeCDict(cdict); + return NULL; + } + + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byRef, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = cdict->customMem; + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_customFree(cdict, cMem); + } + return 0; + } +} + +/*! ZSTD_initStaticCDict_advanced() : + * Generate a digested dictionary in provided memory area. + * workspace: The memory area to emplace the dictionary into. + * Provided pointer must 8-bytes aligned. + * It must outlive dictionary usage. + * workspaceSize: Use ZSTD_estimateCDictSize() + * to determine how large workspace must be. + * cParams : use ZSTD_getCParams() to transform a compression level + * into its relevants cParams. + * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) + * Note : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + */ +const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams) +{ + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); + /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; + ZSTD_CDict* cdict; + ZSTD_CCtx_params params; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + ZSTD_cwksp_move(&cdict->workspace, &ws); + } + + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", + (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); + if (workspaceSize < neededSize) return NULL; + + ZSTD_CCtxParams_init(¶ms, 0); + params.cParams = cParams; + params.useRowMatchFinder = useRowMatchFinder; + cdict->useRowMatchFinder = useRowMatchFinder; + cdict->compressionLevel = ZSTD_NO_CLEVEL; + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + params) )) + return NULL; + + return cdict; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) +{ + assert(cdict != NULL); + return cdict->matchState.cParams; +} + +/*! ZSTD_getDictID_fromCDict() : + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; + return cdict->dictID; +} + +/* ZSTD_compressBegin_usingCDict_internal() : + * Implementation of various ZSTD_compressBegin_usingCDict* functions. + */ +static size_t ZSTD_compressBegin_usingCDict_internal( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); + RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); + /* Initialize the cctxParams from the cdict */ + { + ZSTD_parameters params; + params.fParams = fParams; + params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0 ) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, cdict->compressionLevel); + } + /* Increase window log to fit the entire dictionary and source if the + * source size is known. Limit the increase to 19, which is the + * window log for compression level 1 with the largest source size. + */ + if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19); + U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; + cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog); + } + return ZSTD_compressBegin_internal(cctx, + NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, + cdict, + &cctxParams, pledgedSrcSize, + ZSTDb_not_buffered); +} + + +/* ZSTD_compressBegin_usingCDict_advanced() : + * This function is DEPRECATED. + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); +} + +/* ZSTD_compressBegin_usingCDict() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict); +} + +/*! ZSTD_compress_usingCDict_internal(): + * Implementation of various ZSTD_compress_usingCDict* functions. + */ +static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict_advanced(): + * This function is DEPRECATED. + */ +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression parameters are decided at CDict creation time + * while frame parameters are hardcoded */ +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + + + +/* ****************************************************************** +* Streaming +********************************************************************/ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + DEBUGLOG(3, "ZSTD_createCStream"); + return ZSTD_createCStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticCCtx(workspace, workspaceSize); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ /* CStream and CCtx are now same object */ + return ZSTD_createCCtx_advanced(customMem); +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + return ZSTD_freeCCtx(zcs); /* same object */ +} + + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_CStreamOutSize(void) +{ + return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; +} + +static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) +{ + if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) + return ZSTD_cpm_attachDict; + else + return ZSTD_cpm_noAttachDict; +} + +/* ZSTD_resetCStream(): + * pledgedSrcSize == 0 means "unknown" */ +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +/*! ZSTD_initCStream_internal() : + * Note : for lib/compress only. Used by zstdmt_compress.c. + * Assumption 1 : params are valid + * Assumption 2 : either dict, or cdict, is defined, not both */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_internal"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + zcs->requestedParams = *params; + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if (dict) { + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + } else { + /* Dictionary is cleared if !cdict */ + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + } + return 0; +} + +/* ZSTD_initCStream_usingCDict_advanced() : + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + zcs->requestedParams.fParams = fParams; + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + + +/* ZSTD_initCStream_advanced() : + * pledgedSrcSize must be exact. + * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pss) +{ + /* for compatibility with older programs relying on this behavior. + * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. + * This line will be removed in the future. + */ + U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, ¶ms); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_srcSize"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + return 0; +} + +/*====== Compression ======*/ + +static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + return cctx->blockSize - cctx->stableIn_notConsumed; + } + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered); + { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSize; + return hintInSize; + } +} + +/** ZSTD_compressStream_generic(): + * internal function for all *compressStream*() variants + * @return : hint size for next input to complete ongoing block */ +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) +{ + const char* const istart = (assert(input != NULL), (const char*)input->src); + const char* const iend = (istart != NULL) ? istart + input->size : istart; + const char* ip = (istart != NULL) ? istart + input->pos : istart; + char* const ostart = (assert(output != NULL), (char*)output->dst); + char* const oend = (ostart != NULL) ? ostart + output->size : ostart; + char* op = (ostart != NULL) ? ostart + output->pos : ostart; + U32 someMoreWork = 1; + + /* check expectations */ + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos); + assert(zcs != NULL); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) { + assert(input->pos >= zcs->stableIn_notConsumed); + input->pos -= zcs->stableIn_notConsumed; + ip -= zcs->stableIn_notConsumed; + zcs->stableIn_notConsumed = 0; + } + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize > 0); + } + if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { + assert(zcs->outBuff != NULL); + assert(zcs->outBuffSize > 0); + } + if (input->src == NULL) assert(input->size == 0); + assert(input->pos <= input->size); + if (output->dst == NULL) assert(output->size == 0); + assert(output->pos <= output->size); + assert((U32)flushMode <= (U32)ZSTD_e_end); + + while (someMoreWork) { + switch(zcs->streamStage) + { + case zcss_init: + RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!"); + + case zcss_load: + if ( (flushMode == ZSTD_e_end) + && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ + || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ + && (zcs->inBuffPos == 0) ) { + /* shortcut to compression pass directly into output buffer */ + size_t const cSize = ZSTD_compressEnd_public(zcs, + op, oend-op, ip, iend-ip); + DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); + ip = iend; + op += cSize; + zcs->frameEnded = 1; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + someMoreWork = 0; break; + } + /* complete loading into inBuffer in buffered mode */ + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy( + zcs->inBuff + zcs->inBuffPos, toLoad, + ip, iend-ip); + zcs->inBuffPos += loaded; + if (ip) ip += loaded; + if ( (flushMode == ZSTD_e_continue) + && (zcs->inBuffPos < zcs->inBuffTarget) ) { + /* not enough input to fill full block : stop here */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (zcs->inBuffPos == zcs->inToCompress) ) { + /* empty */ + someMoreWork = 0; break; + } + } else { + assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + if ( (flushMode == ZSTD_e_continue) + && ( (size_t)(iend - ip) < zcs->blockSize) ) { + /* can't compress a full block : stop here */ + zcs->stableIn_notConsumed = (size_t)(iend - ip); + ip = iend; /* pretend to have consumed input */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (ip == iend) ) { + /* empty */ + someMoreWork = 0; break; + } + } + /* compress current block (note : this stage cannot be stopped in the middle) */ + DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); + { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); + void* cDst; + size_t cSize; + size_t oSize = oend-op; + size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSize); + if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) + cDst = op; /* compress into output buffer, to skip flush stage */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + if (inputBuffered) { + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); + cSize = lastBlock ? + ZSTD_compressEnd_public(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize); + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", + (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); + zcs->inToCompress = zcs->inBuffPos; + } else { /* !inputBuffered, hence ZSTD_bm_stable */ + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend); + cSize = lastBlock ? + ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize); + /* Consume the input prior to error checking to mirror buffered mode. */ + if (ip) ip += iSize; + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + if (lastBlock) assert(ip == iend); + } + if (cDst == op) { /* no need to flush */ + op += cSize; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed directly in outBuffer"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + } + break; + } + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->streamStage = zcss_flush; /* pass-through to flush stage */ + } + ZSTD_FALLTHROUGH; + case zcss_flush: + DEBUGLOG(5, "flush stage"); + assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), + zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", + (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); + if (flushed) + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { + /* flush not fully completed, presumably because dst is too small */ + assert(op==oend); + someMoreWork = 0; + break; + } + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed on flush"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + break; + } + zcs->streamStage = zcss_load; + break; + } + + default: /* impossible */ + assert(0); + } + } + + input->pos = ip - istart; + output->pos = op - ostart; + if (zcs->frameEnded) return 0; + return ZSTD_nextInputSizeHint(zcs); +} + +static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers >= 1) { + assert(cctx->mtctx != NULL); + return ZSTDMT_nextInputSizeHint(cctx->mtctx); + } +#endif + return ZSTD_nextInputSizeHint(cctx); + +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , ""); + return ZSTD_nextInputSizeHint_MTorST(zcs); +} + +/* After a compression call set the expected input/output buffer. + * This is validated at the start of the next compression call. + */ +static void +ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input) +{ + DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)"); + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + cctx->expectedInBuffer = *input; + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + cctx->expectedOutBufferSize = output->size - output->pos; + } +} + +/* Validate that the input/output buffers match the expectations set by + * ZSTD_setBufferExpectations. + */ +static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, + ZSTD_outBuffer const* output, + ZSTD_inBuffer const* input, + ZSTD_EndDirective endOp) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + ZSTD_inBuffer const expect = cctx->expectedInBuffer; + if (expect.src != input->src || expect.pos != input->pos) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!"); + } + (void)endOp; + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + size_t const outBufferSize = output->size - output->pos; + if (cctx->expectedOutBufferSize != outBufferSize) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + } + return 0; +} + +static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, + ZSTD_EndDirective endOp, + size_t inSize) +{ + ZSTD_CCtx_params params = cctx->requestedParams; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; + FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + if (cctx->cdict && !cctx->localDict.cdict) { + /* Let the cdict's compression level take priority over the requested params. + * But do not take the cdict's compression level if the "cdict" is actually a localDict + * generated from ZSTD_initLocalDict(). + */ + params.compressionLevel = cctx->cdict->compressionLevel; + } + DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */ + + { size_t const dictSize = prefixDict.dict + ? prefixDict.dictSize + : (cctx->cdict ? cctx->cdict->dictContentSize : 0); + ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); + params.cParams = ZSTD_getCParamsFromCCtxParams( + ¶ms, cctx->pledgedSrcSizePlusOne-1, + dictSize, mode); + } + + params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, ¶ms.cParams); + params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); + params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); + params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences); + params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize); + params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel); + +#ifdef ZSTD_MULTITHREAD + /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */ + RETURN_ERROR_IF( + params.useSequenceProducer == 1 && params.nbWorkers >= 1, + parameter_combination_unsupported, + "External sequence producer isn't supported with nbWorkers >= 1" + ); + + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { + params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ + } + if (params.nbWorkers > 0) { +#if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +#endif + /* mt context creation */ + if (cctx->mtctx == NULL) { + DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", + params.nbWorkers); + cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool); + RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); + } + /* mt compression */ + DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); + FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( + cctx->mtctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); + cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0; + cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize; + cctx->consumedSrcSize = 0; + cctx->producedCSize = 0; + cctx->streamStage = zcss_load; + cctx->appliedParams = params; + } else +#endif /* ZSTD_MULTITHREAD */ + { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast, + cctx->cdict, + ¶ms, pledgedSrcSize, + ZSTDb_buffered) , ""); + assert(cctx->appliedParams.nbWorkers == 0); + cctx->inToCompress = 0; + cctx->inBuffPos = 0; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { + /* for small input: avoid automatic flush on reaching end of block, since + * it would require to add a 3-bytes null block to end frame + */ + cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); + } else { + cctx->inBuffTarget = 0; + } + cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; + cctx->streamStage = zcss_load; + cctx->frameEnded = 0; + } + return 0; +} + +/* @return provides a minimum amount of data remaining to be flushed from internal buffers + */ +size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); + /* check conditions */ + RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); + RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); + RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective"); + assert(cctx != NULL); + + /* transparent initialization stage */ + if (cctx->streamStage == zcss_init) { + size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */ + size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed; + if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */ + && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */ + && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ + if (cctx->stableIn_notConsumed) { /* not the first time */ + /* check stable source guarantees */ + RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer"); + RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos"); + } + /* pretend input was consumed, to give a sense forward progress */ + input->pos = input->size; + /* save stable inBuffer, for later control, and flush/end */ + cctx->expectedInBuffer = *input; + /* but actually input wasn't consumed, so keep track of position from where compression shall resume */ + cctx->stableIn_notConsumed += inputSize; + /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */ + return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */ + } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + } + /* end of transparent initialization stage */ + + FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers"); + /* compression stage */ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + size_t flushMin; + if (cctx->cParamsChanged) { + ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); + cctx->cParamsChanged = 0; + } + if (cctx->stableIn_notConsumed) { + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable); + /* some early data was skipped - make it available for consumption */ + assert(input->pos >= cctx->stableIn_notConsumed); + input->pos -= cctx->stableIn_notConsumed; + cctx->stableIn_notConsumed = 0; + } + for (;;) { + size_t const ipos = input->pos; + size_t const opos = output->pos; + flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); + cctx->consumedSrcSize += (U64)(input->pos - ipos); + cctx->producedCSize += (U64)(output->pos - opos); + if ( ZSTD_isError(flushMin) + || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ + if (flushMin == 0) + ZSTD_CCtx_trace(cctx, 0); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + } + FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed"); + + if (endOp == ZSTD_e_continue) { + /* We only require some progress with ZSTD_e_continue, not maximal progress. + * We're done if we've consumed or produced any bytes, or either buffer is + * full. + */ + if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size) + break; + } else { + assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end); + /* We require maximal progress. We're done when the flush is complete or the + * output buffer is full. + */ + if (flushMin == 0 || output->pos == output->size) + break; + } + } + DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); + /* Either we don't require maximum forward progress, we've finished the + * flush, or we are out of output space. + */ + assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size); + ZSTD_setBufferExpectations(cctx, output, input); + return flushMin; + } +#endif /* ZSTD_MULTITHREAD */ + FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); + DEBUGLOG(5, "completed ZSTD_compressStream2"); + ZSTD_setBufferExpectations(cctx, output, input); + return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ +} + +size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ + { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} + +size_t ZSTD_compress2(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; + ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; + DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + /* Enable stable input/output buffers. */ + cctx->requestedParams.inBufferMode = ZSTD_bm_stable; + cctx->requestedParams.outBufferMode = ZSTD_bm_stable; + { size_t oPos = 0; + size_t iPos = 0; + size_t const result = ZSTD_compressStream2_simpleArgs(cctx, + dst, dstCapacity, &oPos, + src, srcSize, &iPos, + ZSTD_e_end); + /* Reset to the original values. */ + cctx->requestedParams.inBufferMode = originalInBufferMode; + cctx->requestedParams.outBufferMode = originalOutBufferMode; + + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); + if (result != 0) { /* compression not completed, due to lack of output space */ + assert(oPos == dstCapacity); + RETURN_ERROR(dstSize_tooSmall, ""); + } + assert(iPos == srcSize); /* all input is expected consumed */ + return oPos; + } +} + +/* ZSTD_validateSequence() : + * @offCode : is presumed to follow format required by ZSTD_storeSeq() + * @returns a ZSTD error code if sequence is not valid + */ +static size_t +ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch, + size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer) +{ + U32 const windowSize = 1u << windowLog; + /* posInSrc represents the amount of data the decoder would decode up to this point. + * As long as the amount of data decoded is less than or equal to window size, offsets may be + * larger than the total length of output decoded in order to reference the dict, even larger than + * window size. After output surpasses windowSize, we're limited to windowSize offsets again. + */ + size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; + size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4; + RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!"); + /* Validate maxNbSeq is large enough for the given matchLength and minMatch */ + RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch"); + return 0; +} + +/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ +static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) +{ + U32 offBase = OFFSET_TO_OFFBASE(rawOffset); + + if (!ll0 && rawOffset == rep[0]) { + offBase = REPCODE1_TO_OFFBASE; + } else if (rawOffset == rep[1]) { + offBase = REPCODE_TO_OFFBASE(2 - ll0); + } else if (rawOffset == rep[2]) { + offBase = REPCODE_TO_OFFBASE(3 - ll0); + } else if (ll0 && rawOffset == rep[0] - 1) { + offBase = REPCODE3_TO_OFFBASE; + } + return offBase; +} + +size_t +ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, + ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_paramSwitch_e externalRepSearch) +{ + U32 idx = seqPos->idx; + U32 const startIdx = idx; + BYTE const* ip = (BYTE const*)(src); + const BYTE* const iend = ip + blockSize; + repcodes_t updatedRepcodes; + U32 dictSize; + + DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize); + + if (cctx->cdict) { + dictSize = (U32)cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = (U32)cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) { + U32 const litLength = inSeqs[idx].litLength; + U32 const matchLength = inSeqs[idx].matchLength; + U32 offBase; + + if (externalRepSearch == ZSTD_ps_disable) { + offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset); + } else { + U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer), + "Sequence validation failed"); + } + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); + ip += matchLength + litLength; + } + + /* If we skipped repcode search while parsing, we need to update repcodes now */ + assert(externalRepSearch != ZSTD_ps_auto); + assert(idx >= startIdx); + if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) { + U32* const rep = updatedRepcodes.rep; + U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */ + + if (lastSeqIdx >= startIdx + 2) { + rep[2] = inSeqs[lastSeqIdx - 2].offset; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else if (lastSeqIdx == startIdx + 1) { + rep[2] = rep[0]; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else { + assert(lastSeqIdx == startIdx); + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = inSeqs[lastSeqIdx].offset; + } + } + + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + if (inSeqs[idx].litLength) { + DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); + ip += inSeqs[idx].litLength; + seqPos->posInSrc += inSeqs[idx].litLength; + } + RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!"); + seqPos->idx = idx+1; + return 0; +} + +size_t +ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch) +{ + U32 idx = seqPos->idx; + U32 startPosInSequence = seqPos->posInSequence; + U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; + size_t dictSize; + BYTE const* ip = (BYTE const*)(src); + BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ + repcodes_t updatedRepcodes; + U32 bytesAdjustment = 0; + U32 finalMatchSplit = 0; + + /* TODO(embg) support fast parsing mode in noBlockDelim mode */ + (void)externalRepSearch; + + if (cctx->cdict) { + dictSize = cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { + const ZSTD_Sequence currSeq = inSeqs[idx]; + U32 litLength = currSeq.litLength; + U32 matchLength = currSeq.matchLength; + U32 const rawOffset = currSeq.offset; + U32 offBase; + + /* Modify the sequence depending on where endPosInSequence lies */ + if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { + if (startPosInSequence >= litLength) { + startPosInSequence -= litLength; + litLength = 0; + matchLength -= startPosInSequence; + } else { + litLength -= startPosInSequence; + } + /* Move to the next sequence */ + endPosInSequence -= currSeq.litLength + currSeq.matchLength; + startPosInSequence = 0; + } else { + /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence + does not reach the end of the match. So, we have to split the sequence */ + DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", + currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); + if (endPosInSequence > litLength) { + U32 firstHalfMatchLength; + litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; + firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; + if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { + /* Only ever split the match if it is larger than the block size */ + U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; + if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { + /* Move the endPosInSequence backward so that it creates match of minMatch length */ + endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + firstHalfMatchLength -= bytesAdjustment; + } + matchLength = firstHalfMatchLength; + /* Flag that we split the last match - after storing the sequence, exit the loop, + but keep the value of endPosInSequence */ + finalMatchSplit = 1; + } else { + /* Move the position in sequence backwards so that we don't split match, and break to store + * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence + * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so + * would cause the first half of the match to be too small + */ + bytesAdjustment = endPosInSequence - currSeq.litLength; + endPosInSequence = currSeq.litLength; + break; + } + } else { + /* This sequence ends inside the literals, break to store the last literals */ + break; + } + } + /* Check if this offset can be represented with a repcode */ + { U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer), + "Sequence validation failed"); + } + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); + ip += matchLength + litLength; + if (!finalMatchSplit) + idx++; /* Next Sequence */ + } + DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); + seqPos->idx = idx; + seqPos->posInSequence = endPosInSequence; + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + iend -= bytesAdjustment; + if (ip != iend) { + /* Store any last literals */ + U32 lastLLSize = (U32)(iend - ip); + assert(ip <= iend); + DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); + seqPos->posInSrc += lastLLSize; + } + + return bytesAdjustment; +} + +typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); +static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) +{ + ZSTD_sequenceCopier sequenceCopier = NULL; + assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); + if (mode == ZSTD_sf_explicitBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; + } else if (mode == ZSTD_sf_noBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreNoBlockDelim; + } + assert(sequenceCopier != NULL); + return sequenceCopier; +} + +/* Discover the size of next block by searching for the delimiter. + * Note that a block delimiter **must** exist in this mode, + * otherwise it's an input error. + * The block size retrieved will be later compared to ensure it remains within bounds */ +static size_t +blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) +{ + int end = 0; + size_t blockSize = 0; + size_t spos = seqPos.idx; + DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize); + assert(spos <= inSeqsSize); + while (spos < inSeqsSize) { + end = (inSeqs[spos].offset == 0); + blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength; + if (end) { + if (inSeqs[spos].matchLength != 0) + RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0"); + break; + } + spos++; + } + if (!end) + RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter"); + return blockSize; +} + +/* More a "target" block size */ +static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining) +{ + int const lastBlock = (remaining <= blockSize); + return lastBlock ? remaining : blockSize; +} + +static size_t determine_blockSize(ZSTD_sequenceFormat_e mode, + size_t blockSize, size_t remaining, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) +{ + DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); + if (mode == ZSTD_sf_noBlockDelimiters) + return blockSize_noDelimiter(blockSize, remaining); + { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos); + FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters"); + if (explicitBlockSize > blockSize) + RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block"); + if (explicitBlockSize > remaining) + RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source"); + return explicitBlockSize; + } +} + +/* Compress, block-by-block, all of the sequences given. + * + * Returns the cumulative size of all compressed blocks (including their headers), + * otherwise a ZSTD error. + */ +static size_t +ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + size_t cSize = 0; + size_t remaining = srcSize; + ZSTD_sequencePosition seqPos = {0, 0, 0}; + + BYTE const* ip = (BYTE const*)src; + BYTE* op = (BYTE*)dst; + ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + + DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); + /* Special case: empty frame */ + if (remaining == 0) { + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + cSize += ZSTD_blockHeaderSize; + } + + while (remaining) { + size_t compressedSeqsSize; + size_t cBlockSize; + size_t additionalByteAdjustment; + size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, + cctx->blockSize, remaining, + inSeqs, inSeqsSize, seqPos); + U32 const lastBlock = (blockSize == remaining); + FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size"); + assert(blockSize <= remaining); + ZSTD_resetSeqStore(&cctx->seqStore); + DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize); + + additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes); + FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); + blockSize -= additionalByteAdjustment; + + /* If blocks are too small, emit as a nocompress block */ + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); + cSize += cBlockSize; + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + continue; + } + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); + compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, + &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, + &cctx->appliedParams, + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + blockSize, + cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); + DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); + + if (!cctx->isFirstBlock && + ZSTD_maybeRLE(&cctx->seqStore) && + ZSTD_isRLE(ip, blockSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + compressedSeqsSize = 1; + } + + if (compressedSeqsSize == 0) { + /* ZSTD_noCompressBlock writes the block header as well */ + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); + } else if (compressedSeqsSize == 1) { + cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed"); + DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); + } else { + U32 cBlockHeader; + /* Error checking and repcodes update */ + ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + /* Write block header into beginning of block*/ + cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); + MEM_writeLE24(op, cBlockHeader); + cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); + } + + cSize += cBlockSize; + + if (lastBlock) { + break; + } else { + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + } + DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); + } + + DEBUGLOG(4, "cSize final total: %zu", cSize); + return cSize; +} + +size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + BYTE* op = (BYTE*)dst; + size_t cSize = 0; + size_t compressedBlocksSize = 0; + size_t frameHeaderSize = 0; + + /* Transparent initialization stage, same as compressStream2() */ + DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity); + assert(cctx != NULL); + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + /* Begin writing output, starting with frame header */ + frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); + op += frameHeaderSize; + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + if (cctx->appliedParams.fParams.checksumFlag && srcSize) { + XXH64_update(&cctx->xxhState, src, srcSize); + } + /* cSize includes block header size and compressed sequences size */ + compressedBlocksSize = ZSTD_compressSequences_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + src, srcSize); + FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); + cSize += compressedBlocksSize; + dstCapacity -= compressedBlocksSize; + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); + MEM_writeLE32((char*)dst + cSize, checksum); + cSize += 4; + } + + DEBUGLOG(4, "Final compressed size: %zu", cSize); + return cSize; +} + +/*====== Finalize ======*/ + +static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs) +{ + const ZSTD_inBuffer nullInput = { NULL, 0, 0 }; + const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + return stableInput ? zcs->expectedInBuffer : nullInput; +} + +/*! ZSTD_flushStream() : + * @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + input.size = input.pos; /* do not ingest more input during flush */ + return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); + FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed"); + if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ + /* single thread mode : attempt to calculate remaining to flush more precisely */ + { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; + size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); + size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; + DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); + return toFlush; + } +} + + +/*-===== Pre-defined compression levels =====-*/ +#include "clevels.h" + +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } +int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } +int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); + switch (cParams.strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG; + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } + return cParams; +} + +static int ZSTD_dedicatedDictSearch_isSupported( + ZSTD_compressionParameters const* cParams) +{ + return (cParams->strategy >= ZSTD_greedy) + && (cParams->strategy <= ZSTD_lazy2) + && (cParams->hashLog > cParams->chainLog) + && (cParams->chainLog <= 24); +} + +/** + * Reverses the adjustment applied to cparams when enabling dedicated dict + * search. This is used to recover the params set to be used in the working + * context. (Otherwise, those tables would also grow.) + */ +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams) { + switch (cParams->strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; + if (cParams->hashLog < ZSTD_HASHLOG_MIN) { + cParams->hashLog = ZSTD_HASHLOG_MIN; + } + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } +} + +static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + case ZSTD_cpm_createCDict: + break; + case ZSTD_cpm_attachDict: + dictSize = 0; + break; + default: + assert(0); + break; + } + { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; + size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; + return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; + } +} + +/*! ZSTD_getCParams_internal() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. + * Use dictSize == 0 for unknown or unused. + * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); + int row; + DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); + + /* row */ + if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ + else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; + else row = compressionLevel; + + { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; + DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); + /* acceleration factor */ + if (compressionLevel < 0) { + int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); + cp.targetLength = (unsigned)(-clampedCompressionLevel); + } + /* refine parameters based on srcSize & dictSize */ + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); + } +} + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); + DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); + ZSTD_memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + params.fParams.contentSizeFlag = 1; + return params; +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} + +void ZSTD_registerSequenceProducer( + ZSTD_CCtx* zc, void* mState, + ZSTD_sequenceProducer_F* mFinder +) { + if (mFinder != NULL) { + ZSTD_externalMatchCtx emctx; + emctx.mState = mState; + emctx.mFinder = mFinder; + emctx.seqBuffer = NULL; + emctx.seqBufferCapacity = 0; + zc->externalMatchCtx = emctx; + zc->requestedParams.useSequenceProducer = 1; + } else { + ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx)); + zc->requestedParams.useSequenceProducer = 0; + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_internal.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_internal.h new file mode 100644 index 000000000..10f68d010 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_internal.h @@ -0,0 +1,1532 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This header contains definitions + * that shall **only** be used by modules within lib/compress. + */ + +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_internal.h" +#include "zstd_cwksp.h" +#ifdef ZSTD_MULTITHREAD +# include "zstdmt_compress.h" +#endif +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ +#define kSearchStrength 8 +#define HASH_READ_SIZE 8 +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". + It could be confused for a real successor at index "1", if sorted as larger than its predecessor. + It's not a big deal though : candidate will just be sorted again. + Additionally, candidate position 1 will be lost. + But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + + +/*-************************************* +* Context memory management +***************************************/ +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; + +typedef struct ZSTD_prefixDict_s { + const void* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; +} ZSTD_prefixDict; + +typedef struct { + void* dictBuffer; + void const* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; + ZSTD_CDict* cdict; +} ZSTD_localDict; + +typedef struct { + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)]; + HUF_repeat repeatMode; +} ZSTD_hufCTables_t; + +typedef struct { + FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + FSE_repeat offcode_repeatMode; + FSE_repeat matchlength_repeatMode; + FSE_repeat litlength_repeatMode; +} ZSTD_fseCTables_t; + +typedef struct { + ZSTD_hufCTables_t huf; + ZSTD_fseCTables_t fse; +} ZSTD_entropyCTables_t; + +/*********************************************** +* Entropy buffer statistics structs and funcs * +***********************************************/ +/** ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */ +typedef struct { + symbolEncodingType_e hType; + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; + +/** ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */ +typedef struct { + symbolEncodingType_e llType; + symbolEncodingType_e ofType; + symbolEncodingType_e mlType; + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ +} ZSTD_fseCTablesMetadata_t; + +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * @return : 0 on success or error code */ +size_t ZSTD_buildBlockEntropyStats( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); + +/********************************* +* Compression internals structs * +*********************************/ + +typedef struct { + U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */ + U32 len; /* Raw length of match */ +} ZSTD_match_t; + +typedef struct { + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_sequencePosition; + +UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; + +typedef struct { + int price; + U32 off; + U32 mlen; + U32 litlen; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_optimal_t; + +typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; + +typedef struct { + /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ + unsigned* litFreq; /* table of literals statistics, of size 256 */ + unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ + unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ + unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ + ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */ + ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */ + + U32 litSum; /* nb of literals */ + U32 litLengthSum; /* nb of litLength codes */ + U32 matchLengthSum; /* nb of matchLength codes */ + U32 offCodeSum; /* nb of offset codes */ + U32 litSumBasePrice; /* to compare to log2(litfreq) */ + U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */ + U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */ + U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ + ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ + const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ + ZSTD_paramSwitch_e literalCompressionMode; +} optState_t; + +typedef struct { + ZSTD_entropyCTables_t entropy; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_compressedBlockState_t; + +typedef struct { + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more valid data */ + U32 nbOverflowCorrections; /* Number of times overflow correction has run since + * ZSTD_window_init(). Useful for debugging coredumps + * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY. + */ +} ZSTD_window_t; + +#define ZSTD_WINDOW_START_INDEX 2 + +typedef struct ZSTD_matchState_t ZSTD_matchState_t; + +#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */ + +struct ZSTD_matchState_t { + ZSTD_window_t window; /* State for window round buffer management */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. + * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). + * When dict referential is copied into active context (i.e. not attached), + * loadedDictEnd == dictSize, since referential starts from zero. + */ + U32 nextToUpdate; /* index from which to continue table update */ + U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ + + U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ + BYTE* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ + U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ + U64 hashSalt; /* For row-based matchFinder: salts the hash for re-use of tag table */ + U32 hashSaltEntropy; /* For row-based matchFinder: collects entropy for salt generation */ + + U32* hashTable; + U32* hashTable3; + U32* chainTable; + + U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + + int dedicatedDictSearch; /* Indicates whether this matchState is using the + * dedicated dictionary search structure. + */ + optState_t opt; /* optimal parser state */ + const ZSTD_matchState_t* dictMatchState; + ZSTD_compressionParameters cParams; + const rawSeqStore_t* ldmSeqStore; + + /* Controls prefetching in some dictMatchState matchfinders. + * This behavior is controlled from the cctx ms. + * This parameter has no effect in the cdict ms. */ + int prefetchCDictTables; + + /* When == 0, lazy match finders insert every position. + * When != 0, lazy match finders only insert positions they search. + * This allows them to skip much faster over incompressible data, + * at a small cost to compression ratio. + */ + int lazySkipping; +}; + +typedef struct { + ZSTD_compressedBlockState_t* prevCBlock; + ZSTD_compressedBlockState_t* nextCBlock; + ZSTD_matchState_t matchState; +} ZSTD_blockState_t; + +typedef struct { + U32 offset; + U32 checksum; +} ldmEntry_t; + +typedef struct { + BYTE const* split; + U32 hash; + U32 checksum; + ldmEntry_t* bucket; +} ldmMatchCandidate_t; + +#define LDM_BATCH_SIZE 64 + +typedef struct { + ZSTD_window_t window; /* State for the window round buffer management */ + ldmEntry_t* hashTable; + U32 loadedDictEnd; + BYTE* bucketOffsets; /* Next position in bucket to insert entry */ + size_t splitIndices[LDM_BATCH_SIZE]; + ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; +} ldmState_t; + +typedef struct { + ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */ + U32 hashLog; /* Log size of hashTable */ + U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ + U32 minMatchLength; /* Minimum match length */ + U32 hashRateLog; /* Log number of entries to skip */ + U32 windowLog; /* Window log for the LDM */ +} ldmParams_t; + +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + size_t seqIndex; + size_t maxSequences; +} SeqCollector; + +struct ZSTD_CCtx_params_s { + ZSTD_format_e format; + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; + + int compressionLevel; + int forceWindow; /* force back-references to respect limit of + * 1< 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; +} + +/* ZSTD_MLcode() : + * note : mlBase = matchLength - MINMATCH; + * because it's the format it's stored in seqStore->sequences */ +MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +{ + static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + static const U32 ML_deltaCode = 36; + return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; +} + +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +/* ZSTD_noCompressBlock() : + * Writes uncompressed block to dst buffer from given src. + * Returns the size of the block */ +MEM_STATIC size_t +ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +{ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity); + RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, + dstSize_tooSmall, "dst buf too small for uncompressed block"); + MEM_writeLE24(dst, cBlockHeader24); + ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + return ZSTD_blockHeaderSize + srcSize; +} + +MEM_STATIC size_t +ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +{ + BYTE* const op = (BYTE*)dst; + U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, ""); + MEM_writeLE24(op, cBlockHeader); + op[3] = src; + return 4; +} + + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); + return (srcSize >> minlog) + 2; +} + +MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams) +{ + switch (cctxParams->literalCompressionMode) { + case ZSTD_ps_enable: + return 0; + case ZSTD_ps_disable: + return 1; + default: + assert(0 /* impossible: pre-validated */); + ZSTD_FALLTHROUGH; + case ZSTD_ps_auto: + return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + } +} + +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void +ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) +{ + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + + +#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1) +#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2) +#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3) +#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */ +#define OFFSET_TO_OFFBASE(o) (assert((o)>0), o + ZSTD_REP_NUM) +#define OFFBASE_IS_OFFSET(o) ((o) > ZSTD_REP_NUM) +#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM) +#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM) +#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */ + +/*! ZSTD_storeSeq() : + * Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t. + * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE(). + * @matchLength : must be >= MINMATCH + * Allowed to over-read literals up to litLimit. +*/ +HINT_INLINE UNUSED_ATTR void +ZSTD_storeSeq(seqStore_t* seqStorePtr, + size_t litLength, const BYTE* literals, const BYTE* litLimit, + U32 offBase, + size_t matchLength) +{ + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; +#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) + static const BYTE* g_start = NULL; + if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ + { U32 const pos = (U32)((const BYTE*)literals - g_start); + DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u", + pos, (U32)litLength, (U32)matchLength, (U32)offBase); + } +#endif + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + /* copy Literals */ + assert(seqStorePtr->maxNbLit <= 128 KB); + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength>0xFFFF) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_literalLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offBase = offBase; + + /* match Length */ + assert(matchLength >= MINMATCH); + { size_t const mlBase = matchLength - MINMATCH; + if (mlBase>0xFFFF) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_matchLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].mlBase = (U16)mlBase; + } + + seqStorePtr->sequences++; +} + +/* ZSTD_updateRep() : + * updates in-place @rep (array of repeat offsets) + * @offBase : sum-type, using numeric representation of ZSTD_storeSeq() + */ +MEM_STATIC void +ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + if (OFFBASE_IS_OFFSET(offBase)) { /* full offset */ + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = OFFBASE_TO_OFFSET(offBase); + } else { /* repcode */ + U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + rep[1] = rep[0]; + rep[0] = currentOffset; + } else { /* repCode == 0 */ + /* nothing to do */ + } + } +} + +typedef struct repcodes_s { + U32 rep[3]; +} repcodes_t; + +MEM_STATIC repcodes_t +ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + repcodes_t newReps; + ZSTD_memcpy(&newReps, rep, sizeof(newReps)); + ZSTD_updateRep(newReps.rep, offBase, ll0); + return newReps; +} + + +/*-************************************* +* Match length counter +***************************************/ +MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + if (pIn < pInLoopLimit) { + { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) return ZSTD_NbCommonBytes(diff); } + pIn+=sizeof(size_t); pMatch+=sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } } + if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h, 0); } /* only in zstd_opt.h */ +MEM_STATIC size_t ZSTD_hash3PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash3(MEM_readLE32(ptr), h, s); } + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h, U32 s) { assert(h <= 32); return ((u * prime4bytes) ^ s) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h, 0); } +static size_t ZSTD_hash4PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash4(MEM_readLE32(ptr), h, s); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-40)) * prime5bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash5PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash5(MEM_readLE64(p), h, s); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-48)) * prime6bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash6PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash6(MEM_readLE64(p), h, s); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-56)) * prime7bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash7PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash7(MEM_readLE64(p), h, s); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u) * prime8bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash8PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash8(MEM_readLE64(p), h, s); } + + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtrSalted(const void* p, U32 hBits, U32 mls, const U64 hashSalt) { + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4PtrS(p, hBits, (U32)hashSalt); + case 5: return ZSTD_hash5PtrS(p, hBits, hashSalt); + case 6: return ZSTD_hash6PtrS(p, hBits, hashSalt); + case 7: return ZSTD_hash7PtrS(p, hBits, hashSalt); + case 8: return ZSTD_hash8PtrS(p, hBits, hashSalt); + } +} + + +/** ZSTD_ipow() : + * Return base^exponent. + */ +static U64 ZSTD_ipow(U64 base, U64 exponent) +{ + U64 power = 1; + while (exponent) { + if (exponent & 1) power *= base; + exponent >>= 1; + base *= base; + } + return power; +} + +#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 + +/** ZSTD_rollingHash_append() : + * Add the buffer to the hash value. + */ +static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +{ + BYTE const* istart = (BYTE const*)buf; + size_t pos; + for (pos = 0; pos < size; ++pos) { + hash *= prime8bytes; + hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; + } + return hash; +} + +/** ZSTD_rollingHash_compute() : + * Compute the rolling hash value of the buffer. + */ +MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) +{ + return ZSTD_rollingHash_append(0, buf, size); +} + +/** ZSTD_rollingHash_primePower() : + * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash + * over a window of length bytes. + */ +MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) +{ + return ZSTD_ipow(prime8bytes, length - 1); +} + +/** ZSTD_rollingHash_rotate() : + * Rotate the rolling hash by one byte. + */ +MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) +{ + hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; + hash *= prime8bytes; + hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; + return hash; +} + +/*-************************************* +* Round buffer management +***************************************/ +#if (ZSTD_WINDOWLOG_MAX_64 > 31) +# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" +#endif +/* Max current allowed */ +#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) +/* Maximum chunk size before overflow correction needs to be called again */ +#define ZSTD_CHUNKSIZE_MAX \ + ( ((U32)-1) /* Maximum ending current index */ \ + - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */ + +/** + * ZSTD_window_clear(): + * Clears the window containing the history by simply setting it to empty. + */ +MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) +{ + size_t const endT = (size_t)(window->nextSrc - window->base); + U32 const end = (U32)endT; + + window->lowLimit = end; + window->dictLimit = end; +} + +MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window) +{ + return window.dictLimit == ZSTD_WINDOW_START_INDEX && + window.lowLimit == ZSTD_WINDOW_START_INDEX && + (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX; +} + +/** + * ZSTD_window_hasExtDict(): + * Returns non-zero if the window has a non-empty extDict. + */ +MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) +{ + return window.lowLimit < window.dictLimit; +} + +/** + * ZSTD_matchState_dictMode(): + * Inspects the provided matchState and figures out what dictMode should be + * passed to the compressor. + */ +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) +{ + return ZSTD_window_hasExtDict(ms->window) ? + ZSTD_extDict : + ms->dictMatchState != NULL ? + (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : + ZSTD_noDict; +} + +/* Defining this macro to non-zero tells zstd to run the overflow correction + * code much more frequently. This is very inefficient, and should only be + * used for tests and fuzzers. + */ +#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY +# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1 +# else +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0 +# endif +#endif + +/** + * ZSTD_window_canOverflowCorrect(): + * Returns non-zero if the indices are large enough for overflow correction + * to work correctly without impacting compression ratio. + */ +MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src) +{ + U32 const cycleSize = 1u << cycleLog; + U32 const curr = (U32)((BYTE const*)src - window.base); + U32 const minIndexToOverflowCorrect = cycleSize + + MAX(maxDist, cycleSize) + + ZSTD_WINDOW_START_INDEX; + + /* Adjust the min index to backoff the overflow correction frequency, + * so we don't waste too much CPU in overflow correction. If this + * computation overflows we don't really care, we just need to make + * sure it is at least minIndexToOverflowCorrect. + */ + U32 const adjustment = window.nbOverflowCorrections + 1; + U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, + minIndexToOverflowCorrect); + U32 const indexLargeEnough = curr > adjustedIndex; + + /* Only overflow correct early if the dictionary is invalidated already, + * so we don't hurt compression ratio. + */ + U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; + + return indexLargeEnough && dictionaryInvalidated; +} + +/** + * ZSTD_window_needOverflowCorrection(): + * Returns non-zero if the indices are getting too large and need overflow + * protection. + */ +MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src, + void const* srcEnd) +{ + U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { + return 1; + } + } + return curr > ZSTD_CURRENT_MAX; +} + +/** + * ZSTD_window_correctOverflow(): + * Reduces the indices to protect from index overflow. + * Returns the correction made to the indices, which must be applied to every + * stored index. + * + * The least significant cycleLog bits of the indices must remain the same, + * which may be 0. Every index up to maxDist in the past must be valid. + */ +MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, + U32 maxDist, void const* src) +{ + /* preemptive overflow correction: + * 1. correction is large enough: + * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) + * > 1<<29 + * + * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: + * After correction, current is less than (1<base < 1<<32. + * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); + U32 const currentCycle = curr & cycleMask; + /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */ + U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX + ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX) + : 0; + U32 const newCurrent = currentCycle + + currentCycleCorrection + + MAX(maxDist, cycleSize); + U32 const correction = curr - newCurrent; + /* maxDist must be a power of two so that: + * (newCurrent & cycleMask) == (curr & cycleMask) + * This is required to not corrupt the chains / binary tree. + */ + assert((maxDist & (maxDist - 1)) == 0); + assert((curr & cycleMask) == (newCurrent & cycleMask)); + assert(curr > newCurrent); + if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1<<28); + } + + window->base += correction; + window->dictBase += correction; + if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->lowLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->lowLimit -= correction; + } + if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->dictLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->dictLimit -= correction; + } + + /* Ensure we can still reference the full window. */ + assert(newCurrent >= maxDist); + assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX); + /* Ensure that lowLimit and dictLimit didn't underflow. */ + assert(window->lowLimit <= newCurrent); + assert(window->dictLimit <= newCurrent); + + ++window->nbOverflowCorrections; + + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, + window->lowLimit); + return correction; +} + +/** + * ZSTD_window_enforceMaxDist(): + * Updates lowLimit so that: + * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd + * + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. + * + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. + * + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. + * + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. + */ +MEM_STATIC void +ZSTD_window_enforceMaxDist(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ + if (blockEndIdx > maxDist + loadedDictEnd) { + U32 const newLowLimit = blockEndIdx - maxDist; + if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; + if (window->dictLimit < window->lowLimit) { + DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u", + (unsigned)window->dictLimit, (unsigned)window->lowLimit); + window->dictLimit = window->lowLimit; + } + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. + * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) + * loadedDictEnd uses same referential as window->base + * maxDist is the window size */ +MEM_STATIC void +ZSTD_checkDictValidity(const ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + assert(loadedDictEndPtr != NULL); + assert(dictMatchStatePtr != NULL); + { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = *loadedDictEndPtr; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(blockEndIdx >= loadedDictEnd); + + if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) { + /* On reaching window size, dictionaries are invalidated. + * For simplification, if window size is reached anywhere within next block, + * the dictionary is invalidated for the full block. + * + * We also have to invalidate the dictionary if ZSTD_window_update() has detected + * non-contiguous segments, which means that loadedDictEnd != window->dictLimit. + * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use + * dictMatchState, so setting it to NULL is not a problem. + */ + DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); + *loadedDictEndPtr = 0; + *dictMatchStatePtr = NULL; + } else { + if (*loadedDictEndPtr != 0) { + DEBUGLOG(6, "dictionary considered valid for current block"); + } } } +} + +MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { + ZSTD_memset(window, 0, sizeof(*window)); + window->base = (BYTE const*)" "; + window->dictBase = (BYTE const*)" "; + ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */ + window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */ + window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */ + window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */ + window->nbOverflowCorrections = 0; +} + +/** + * ZSTD_window_update(): + * Updates the window by appending [src, src + srcSize) to the window. + * If it is not contiguous, the current prefix becomes the extDict, and we + * forget about the extDict. Handles overlap of the prefix and extDict. + * Returns non-zero if the segment is contiguous. + */ +MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, + void const* src, size_t srcSize, + int forceNonContiguous) +{ + BYTE const* const ip = (BYTE const*)src; + U32 contiguous = 1; + DEBUGLOG(5, "ZSTD_window_update"); + if (srcSize == 0) + return contiguous; + assert(window->base != NULL); + assert(window->dictBase != NULL); + /* Check if blocks follow each other */ + if (src != window->nextSrc || forceNonContiguous) { + /* not contiguous */ + size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); + DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); + window->lowLimit = window->dictLimit; + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + window->dictLimit = (U32)distanceFromBase; + window->dictBase = window->base; + window->base = ip - distanceFromBase; + /* ms->nextToUpdate = window->dictLimit; */ + if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */ + contiguous = 0; + } + window->nextSrc = ip + srcSize; + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ( (ip+srcSize > window->dictBase + window->lowLimit) + & (ip < window->dictBase + window->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + window->lowLimit = lowLimitMax; + DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); + } + return contiguous; +} + +/** + * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary + * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't + * valid for the entire block. So this check is sufficient to find the lowest valid match index. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + +/** + * Returns the lowest allowed match index in the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.dictLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When computing the lowest prefix index we need to take the dictionary into account to handle + * the edge case where the dictionary and the source are contiguous in memory. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + + + +/* debug functions */ +#if (DEBUGLEVEL>=2) + +MEM_STATIC double ZSTD_fWeight(U32 rawStat) +{ + U32 const fp_accuracy = 8; + U32 const fp_multiplier = (1 << fp_accuracy); + U32 const newStat = rawStat + 1; + U32 const hb = ZSTD_highbit32(newStat); + U32 const BWeight = hb * fp_multiplier; + U32 const FWeight = (newStat << fp_accuracy) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + fp_accuracy < 31); + return (double)weight / fp_multiplier; +} + +/* display a table content, + * listing each element, its frequency, and its predicted bit cost */ +MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) +{ + unsigned u, sum; + for (u=0, sum=0; u<=max; u++) sum += table[u]; + DEBUGLOG(2, "total nb elts: %u", sum); + for (u=0; u<=max; u++) { + DEBUGLOG(2, "%2u: %5u (%.2f)", + u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) ); + } +} + +#endif + +/* Short Cache */ + +/* Normally, zstd matchfinders follow this flow: + * 1. Compute hash at ip + * 2. Load index from hashTable[hash] + * 3. Check if *ip == *(base + index) + * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss. + * + * Short cache is an optimization which allows us to avoid step 3 most of the time + * when the data doesn't actually match. With short cache, the flow becomes: + * 1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip. + * 2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works. + * 3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue. + * + * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to + * dictMatchState matchfinders. + */ +#define ZSTD_SHORT_CACHE_TAG_BITS 8 +#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1) + +/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable. + * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */ +MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) { + size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK); + assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0); + hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag; +} + +/* Helper function for short cache matchfinders. + * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */ +MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) { + U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK; + U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK; + return tag1 == tag2; +} + +#if defined (__cplusplus) +} +#endif + +/* =============================================================== + * Shared internal declarations + * These prototypes may be called from sources not in lib/compress + * =============================================================== */ + +/* ZSTD_loadCEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * return : size of dictionary header (size of magic number + dict ID + entropy tables) + * assumptions : magic number supposed already checked + * and dictSize >= 8 */ +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize); + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); + +/* ============================================================== + * Private declarations + * These prototypes shall only be called from within lib/compress + * ============================================================== */ + +/* ZSTD_getCParamsFromCCtxParams() : + * cParams are built depending on compressionLevel, src size hints, + * LDM and manually set compression parameters. + * Note: srcSizeHint == 0 means 0! + */ +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +/*! ZSTD_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); + +void ZSTD_resetSeqStore(seqStore_t* ssPtr); + +/*! ZSTD_getCParamsFromCDict() : + * as the name implies */ +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); + +/* ZSTD_compressBegin_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize); + +/* ZSTD_compress_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params); + + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small ( 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); + +/** ZSTD_CCtx_trace() : + * Trace the end of a compression call. + */ +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); + +/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of + * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. + * Note that the block delimiter must include the last literals of the block. + */ +size_t +ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, + ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); + +/* Returns the number of bytes to move the current read position back by. + * Only non-zero if we ended up splitting a sequence. + * Otherwise, it may return a ZSTD error if something went wrong. + * + * This function will attempt to scan through blockSize bytes + * represented by the sequences in @inSeqs, + * storing any (partial) sequences. + * + * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to + * avoid splitting a match, or to avoid splitting a match such that it would produce a match + * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + */ +size_t +ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); + + +/* =============================================================== + * Deprecated definitions that are still used internally to avoid + * deprecation warnings. These functions are exactly equivalent to + * their public variants, but avoid the deprecation warnings. + * =============================================================== */ + +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +#endif /* ZSTD_COMPRESS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.c new file mode 100644 index 000000000..bfd4f11ab --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_literals.h" + + +/* ************************************************************** +* Debug Traces +****************************************************************/ +#if DEBUGLEVEL >= 2 + +static size_t showHexa(const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*)src; + size_t u; + for (u=0; u31) + (srcSize>4095); + + DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity); + + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ZSTD_memcpy(ostart + flSize, src, srcSize); + DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); + return srcSize + flSize; +} + +static int allBytesIdentical(const void* src, size_t srcSize) +{ + assert(srcSize >= 1); + assert(src != NULL); + { const BYTE b = ((const BYTE*)src)[0]; + size_t p; + for (p=1; p31) + (srcSize>4095); + + assert(dstCapacity >= 4); (void)dstCapacity; + assert(allBytesIdentical(src, srcSize)); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1); + return flSize+1; +} + +/* ZSTD_minLiteralsToCompress() : + * returns minimal amount of literals + * for literal compression to even be attempted. + * Minimum is made tighter as compression strategy increases. + */ +static size_t +ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat) +{ + assert((int)strategy >= 0); + assert((int)strategy <= 9); + /* btultra2 : min 8 bytes; + * then 2x larger for each successive compression strategy + * max threshold 64 bytes */ + { int const shift = MIN(9-(int)strategy, 3); + size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift; + DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc); + return mintc; + } +} + +size_t ZSTD_compressLiterals ( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, + int disableLiteralCompression, + int suspectUncompressible, + int bmi2) +{ + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)", + disableLiteralCompression, (U32)srcSize, dstCapacity); + + DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize)); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* if too small, don't even attempt compression (speed opt) */ + if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); + { HUF_repeat repeat = prevHuf->repeatMode; + int const flags = 0 + | (bmi2 ? HUF_flags_bmi2 : 0) + | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0) + | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0) + | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0); + + typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int); + huf_compress_f huf_compress; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat; + cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize, + src, srcSize, + HUF_SYMBOLVALUE_MAX, LitHufLog, + entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, + &repeat, flags); + DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + DEBUGLOG(5, "reusing statistics from previous huffman block"); + hType = set_repeat; + } + } + + { size_t const minGain = ZSTD_minGain(srcSize, strategy); + if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } } + if (cLitSize==1) { + /* A return value of 1 signals that the alphabet consists of a single symbol. + * However, in some rare circumstances, it could be the compressed size (a single byte). + * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`. + * (it's also necessary to not generate statistics). + * Therefore, in such a case, actively check that all bytes are identical. */ + if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize)); + return lhSize+cLitSize; +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.h new file mode 100644 index 000000000..b060c8ad2 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_LITERALS_H +#define ZSTD_COMPRESS_LITERALS_H + +#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */ + + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* ZSTD_compressRleLiteralsBlock() : + * Conditions : + * - All bytes in @src are identical + * - dstCapacity >= 4 */ +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* ZSTD_compressLiterals(): + * @entropyWorkspace: must be aligned on 4-bytes boundaries + * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE + * @suspectUncompressible: sampling checks, to potentially skip huffman coding + */ +size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + int suspectUncompressible, + int bmi2); + +#endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.c new file mode 100644 index 000000000..8872d4d35 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.c @@ -0,0 +1,442 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_sequences.h" + +/** + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabilityLog256[256] = { + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, + 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, + 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, + 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, + 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, + 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, + 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, + 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, + 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, + 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, + 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, + 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, + 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, + 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, + 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, + 5, 4, 2, 1, +}; + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/** + * Returns true if we should use ncount=-1 else we should + * use ncount=1 for low probability symbols instead. + */ +static unsigned ZSTD_useLowProbCount(size_t const nbSeq) +{ + /* Heuristic: This should cover most blocks <= 16K and + * start to fade out after 16K to about 32K depending on + * compressibility. + */ + return nbSeq >= 2048; +} + +/** + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, + size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +/** + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + + assert(total > 0); + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabilityLog256[norm]; + } + return cost >> 8; +} + +/** + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { + DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", + ZSTD_getFSEMaxSymbolValue(ctable), max); + return ERROR(GENERIC); + } + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + if (bitCost >= badCost) { + DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); + return ERROR(GENERIC); + } + cost += (size_t)count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/** + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabilityLog256[norm256]; + } + return cost >> 8; +} + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or fewer symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ( (*repeatMode == FSE_repeat_valid) + && (nbSeq < staticFse_nbSeq_max) ) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ( (nbSeq < dynamicFse_nbSeq_min) + || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +typedef struct { + S16 norm[MaxSeq + 1]; + U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)]; +} ZSTD_BuildCTableWksp; + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); + *op = codeTable[0]; + return 1; + case set_repeat: + ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ + return 0; + case set_compressed: { + ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); + (void)entropyWorkspaceSize; + FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed"); + assert(oend >= op); + { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */ + FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed"); + return NCountSize; + } + } + default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].mlBase, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offBase, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offBase >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + +#endif + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.h new file mode 100644 index 000000000..4a3a05da9 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_SEQUENCES_H +#define ZSTD_COMPRESS_SEQUENCES_H + +#include "../common/fse.h" /* FSE_repeat, FSE_CTable */ +#include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_defaultPolicy_e; + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy); + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize); + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max); + +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max); +#endif /* ZSTD_COMPRESS_SEQUENCES_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.c new file mode 100644 index 000000000..638c4acbe --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.c @@ -0,0 +1,577 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_superblock.h" + +#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */ +#include "hist.h" /* HIST_countFast_wksp */ +#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */ +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" + +/** ZSTD_compressSubBlock_literal() : + * Compresses literals section for a sub-block. + * When we have to write the Huffman table we will sometimes choose a header + * size larger than necessary. This is because we have to pick the header size + * before we know the table size + compressed size, so we have a bound on the + * table size. If we guessed incorrectly, we fall back to uncompressed literals. + * + * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded + * in writing the header, otherwise it is set to 0. + * + * hufMetadata->hType has literals block type info. + * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. + * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. + * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block + * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block + * and the following sub-blocks' literals sections will be Treeless_Literals_Block. + * @return : compressed size of literals section of a sub-block + * Or 0 if unable to compress. + * Or error code */ +static size_t +ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + size_t const header = writeEntropy ? 200 : 0; + size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart + lhSize; + U32 const singleStream = lhSize == 3; + symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; + size_t cLitSize = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); + + *entropyWritten = 0; + if (litSize == 0 || hufMetadata->hType == set_basic) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } else if (hufMetadata->hType == set_rle) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); + return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); + } + + assert(litSize > 0); + assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); + + if (writeEntropy && hufMetadata->hType == set_compressed) { + ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); + op += hufMetadata->hufDesSize; + cLitSize += hufMetadata->hufDesSize; + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); + } + + { int const flags = bmi2 ? HUF_flags_bmi2 : 0; + const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags) + : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags); + op += cSize; + cLitSize += cSize; + if (cSize == 0 || ERR_isError(cSize)) { + DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); + return 0; + } + /* If we expand and we aren't writing a header then emit uncompressed */ + if (!writeEntropy && cLitSize >= litSize) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + /* If we are writing headers then allow expansion that doesn't change our header size. */ + if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { + assert(cLitSize > litSize); + DEBUGLOG(5, "Literals expanded beyond allowed header size"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + *entropyWritten = 1; + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); + return op-ostart; +} + +static size_t +ZSTD_seqDecompressedSize(seqStore_t const* seqStore, + const seqDef* sequences, size_t nbSeq, + size_t litSize, int lastSequence) +{ + const seqDef* const sstart = sequences; + const seqDef* const send = sequences + nbSeq; + const seqDef* sp = sstart; + size_t matchLengthSum = 0; + size_t litLengthSum = 0; + (void)(litLengthSum); /* suppress unused variable warning on some environments */ + while (send-sp > 0) { + ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp); + litLengthSum += seqLen.litLength; + matchLengthSum += seqLen.matchLength; + sp++; + } + assert(litLengthSum <= litSize); + if (!lastSequence) { + assert(litLengthSum == litSize); + } + return matchLengthSum + litSize; +} + +/** ZSTD_compressSubBlock_sequences() : + * Compresses sequences section for a sub-block. + * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have + * symbol compression modes for the super-block. + * The first successfully compressed block will have these in its header. + * We set entropyWritten=1 when we succeed in compressing the sequences. + * The following sub-blocks will always have repeat mode. + * @return : compressed size of sequences section of a sub-block + * Or 0 if it is unable to compress + * Or error code. */ +static size_t +ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + BYTE* seqHead; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); + + *entropyWritten = 0; + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, ""); + if (nbSeq < 0x7F) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else + op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) { + return op - ostart; + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); + + if (writeEntropy) { + const U32 LLtype = fseMetadata->llType; + const U32 Offtype = fseMetadata->ofType; + const U32 MLtype = fseMetadata->mlType; + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); + op += fseMetadata->fseTablesSize; + } else { + const U32 repeat = set_repeat; + *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, oend - op, + fseTables->matchlengthCTable, mlCode, + fseTables->offcodeCTable, ofCode, + fseTables->litlengthCTable, llCode, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(fseMetadata->lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } +#endif + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); + } + + /* zstd versions <= 1.4.0 mistakenly report error when + * sequences section body size is less than 3 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1664. + * This can happen when the previous sequences section block is compressed + * with rle mode and the current block's sequences section is compressed + * with repeat mode where sequences section body size can be 1 byte. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (op-seqHead < 4) { + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " + "an uncompressed block when sequences are < 4 bytes"); + return 0; + } +#endif + + *entropyWritten = 1; + return op - ostart; +} + +/** ZSTD_compressSubBlock() : + * Compresses a single sub-block. + * @return : compressed size of the sub-block + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* literals, size_t litSize, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, + int writeLitEntropy, int writeSeqEntropy, + int* litEntropyWritten, int* seqEntropyWritten, + U32 lastBlock) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart + ZSTD_blockHeaderSize; + DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", + litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); + { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, + &entropyMetadata->hufMetadata, literals, litSize, + op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); + FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); + if (cLitSize == 0) return 0; + op += cLitSize; + } + { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, + &entropyMetadata->fseMetadata, + sequences, nbSeq, + llCode, mlCode, ofCode, + cctxParams, + op, oend-op, + bmi2, writeSeqEntropy, seqEntropyWritten); + FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); + if (cSeqSize == 0) return 0; + op += cSeqSize; + } + /* Write block header */ + { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize; + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(ostart, cBlockHeader24); + } + return op-ostart; +} + +static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = 255; + size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, unsigned maxCode, + size_t nbSeq, const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + cSymbolTypeSizeEstimateInBits = max <= defaultMax + ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) + : ERROR(GENERIC); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits / 8; +} + +static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + size_t cSeqSizeEstimate = 0; + if (nbSeq == 0) return sequencesSectionHeaderSize; + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, + nbSeq, fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, + nbSeq, fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, + nbSeq, fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) { + size_t cSizeEstimate = 0; + cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return cSizeEstimate + ZSTD_blockHeaderSize; +} + +static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) +{ + if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) + return 1; + if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) + return 1; + if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) + return 1; + return 0; +} + +/** ZSTD_compressSubBlock_multi() : + * Breaks super-block into multiple sub-blocks and compresses them. + * Entropy will be written to the first block. + * The following blocks will use repeat mode to compress. + * All sub-blocks are compressed blocks (no raw or rle blocks). + * @return : compressed size of the super block (which is multiple ZSTD blocks) + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, + const ZSTD_compressedBlockState_t* prevCBlock, + ZSTD_compressedBlockState_t* nextCBlock, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const int bmi2, U32 lastBlock, + void* workspace, size_t wkspSize) +{ + const seqDef* const sstart = seqStorePtr->sequencesStart; + const seqDef* const send = seqStorePtr->sequences; + const seqDef* sp = sstart; + const BYTE* const lstart = seqStorePtr->litStart; + const BYTE* const lend = seqStorePtr->lit; + const BYTE* lp = lstart; + BYTE const* ip = (BYTE const*)src; + BYTE const* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + const BYTE* llCodePtr = seqStorePtr->llCode; + const BYTE* mlCodePtr = seqStorePtr->mlCode; + const BYTE* ofCodePtr = seqStorePtr->ofCode; + size_t targetCBlockSize = cctxParams->targetCBlockSize; + size_t litSize, seqCount; + int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed; + int writeSeqEntropy = 1; + int lastSequence = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)", + (unsigned)(lend-lp), (unsigned)(send-sstart)); + + litSize = 0; + seqCount = 0; + do { + size_t cBlockSizeEstimate = 0; + if (sstart == send) { + lastSequence = 1; + } else { + const seqDef* const sequence = sp + seqCount; + lastSequence = sequence == send - 1; + litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength; + seqCount++; + } + if (lastSequence) { + assert(lp <= lend); + assert(litSize <= (size_t)(lend - lp)); + litSize = (size_t)(lend - lp); + } + /* I think there is an optimization opportunity here. + * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful + * since it recalculates estimate from scratch. + * For example, it would recount literal distribution and symbol codes every time. + */ + cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount, + &nextCBlock->entropy, entropyMetadata, + workspace, wkspSize, writeLitEntropy, writeSeqEntropy); + if (cBlockSizeEstimate > targetCBlockSize || lastSequence) { + int litEntropyWritten = 0; + int seqEntropyWritten = 0; + const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence); + const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, oend-op, + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + lastBlock && lastSequence); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Committed the sub-block"); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + sp += seqCount; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + litSize = 0; + seqCount = 0; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + } + } + } while (!lastSequence); + if (writeLitEntropy) { + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); + ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); + } + if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { + /* If we haven't written our entropy tables, then we've violated our contract and + * must emit an uncompressed block. + */ + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten"); + return 0; + } + if (ip < iend) { + size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock); + DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip)); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + assert(cSize != 0); + op += cSize; + /* We have to regenerate the repcodes because we've skipped some sequences */ + if (sp < send) { + seqDef const* seq; + repcodes_t rep; + ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); + for (seq = sstart; seq < sp; ++seq) { + ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + } + ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); + } + } + DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); + return op-ostart; +} + +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock) { + ZSTD_entropyCTablesMetadata_t entropyMetadata; + + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + &entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + + return ZSTD_compressSubBlock_multi(&zc->seqStore, + zc->blockState.prevCBlock, + zc->blockState.nextCBlock, + &entropyMetadata, + &zc->appliedParams, + dst, dstCapacity, + src, srcSize, + zc->bmi2, lastBlock, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.h new file mode 100644 index 000000000..8e494f0d5 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_ADVANCED_H +#define ZSTD_COMPRESS_ADVANCED_H + +/*-************************************* +* Dependencies +***************************************/ + +#include "../zstd.h" /* ZSTD_CCtx */ + +/*-************************************* +* Target Compressed Block Size +***************************************/ + +/* ZSTD_compressSuperBlock() : + * Used to compress a super block when targetCBlockSize is being used. + * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock); + +#endif /* ZSTD_COMPRESS_ADVANCED_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h new file mode 100644 index 000000000..cc7fb1c71 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h @@ -0,0 +1,742 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "../common/zstd_internal.h" +#include "../common/portability_macros.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ + +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#endif + + +/* Set our tables and aligneds to align by 64 bytes */ +#define ZSTD_CWKSP_ALIGNMENT_BYTES 64 + +/*-************************************* +* Structures +***************************************/ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_aligned_init_once, + ZSTD_cwksp_alloc_aligned, + ZSTD_cwksp_alloc_buffers +} ZSTD_cwksp_alloc_phase_e; + +/** + * Used to describe whether the workspace is statically allocated (and will not + * necessarily ever be freed), or if it's dynamically allocated and we can + * expect a well-formed caller to free this. + */ +typedef enum { + ZSTD_cwksp_dynamic_alloc, + ZSTD_cwksp_static_alloc +} ZSTD_cwksp_static_alloc_e; + +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each internal datastructure: + * + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ->] free space [<- buffers][<- aligned][<- init once] + * + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_customFree{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. These tables are 64-byte aligned. + * + * - Init once: these buffers require to be initialized at least once before + * use. They should be used when we want to skip memory initialization + * while not triggering memory checkers (like Valgrind) when reading from + * from this memory without writing to it first. + * These buffers should be used carefully as they might contain data + * from previous compressions. + * Buffers are aligned to 64 bytes. + * + * - Aligned: these buffers don't require any initialization before they're + * used. The user of the buffer should make sure they write into a buffer + * location before reading from it. + * Buffers are aligned to 64 bytes. + * + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Init once / Tables + * 3. Aligned / Tables + * 4. Buffers / Tables + * + * Attempts to reserve objects of different types out of order will fail. + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* tableValidEnd; + void* allocStart; + void* initOnceStart; + + BYTE allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; + ZSTD_cwksp_static_alloc_e isStatic; +} ZSTD_cwksp; + +/*-************************************* +* Functions +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws); + +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); + assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws)); + assert(ws->workspace <= ws->initOnceStart); +#if ZSTD_MEMORY_SANITIZER + { + intptr_t const offset = __msan_test_shadow(ws->initOnceStart, + (U8*)ZSTD_cwksp_initialAllocStart(ws) - (U8*)ws->initOnceStart); +#if defined(ZSTD_MSAN_PRINT) + if(offset!=-1) { + __msan_print_shadow((U8*)ws->initOnceStart + offset - 8, 32); + } +#endif + assert(offset==-1); + }; +#endif +} + +/** + * Align must be a power of 2. + */ +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/** + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. + * + * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size(). + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { + if (size == 0) + return 0; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + +/** + * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes. + * Used to determine the number of bytes required for a given "aligned". + */ +MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) { + return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES)); +} + +/** + * Returns the amount of additional space the cwksp must allocate + * for internal purposes (currently only alignment). + */ +MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) { + /* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES + * bytes to align the beginning of tables section and end of buffers; + */ + size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; + return slackSpace; +} + + +/** + * Return the number of additional bytes required to align a pointer to the given number of bytes. + * alignBytes must be a power of two. + */ +MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) { + size_t const alignBytesMask = alignBytes - 1; + size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask; + assert((alignBytes & alignBytesMask) == 0); + assert(bytes < alignBytes); + return bytes; +} + +/** + * Returns the initial value for allocStart which is used to determine the position from + * which we can allocate from the end of the workspace. + */ +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) { + return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1)); +} + +/** + * Internal function. Do not use directly. + * Reserves the given number of bytes within the aligned/buffer segment of the wksp, + * which counts from the end of the wksp (as opposed to the object/table segment). + * + * Returns a pointer to the beginning of that space. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes) +{ + void* const alloc = (BYTE*)ws->allocStart - bytes; + void* const bottom = ws->tableEnd; + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(alloc >= bottom); + if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + /* the area is reserved from the end of wksp. + * If it overlaps with tableValidEnd, it voids guarantees on values' range */ + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; + } + ws->allocStart = alloc; + return alloc; +} + +/** + * Moves the cwksp to the next phase, and does any necessary allocations. + * cwksp initialization must necessarily go through each phase in order. + * Returns a 0 on success, or zstd error + */ +MEM_STATIC size_t +ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) +{ + assert(phase >= ws->phase); + if (phase > ws->phase) { + /* Going from allocating objects to allocating initOnce / tables */ + if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once && + phase >= ZSTD_cwksp_alloc_aligned_init_once) { + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + + { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */ + void *const alloc = ws->objectEnd; + size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES); + void *const objectEnd = (BYTE *) alloc + bytesToAlign; + DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign); + RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation, + "table phase - alignment initial allocation failed!"); + ws->objectEnd = objectEnd; + ws->tableEnd = objectEnd; /* table area starts being empty */ + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + } + } + ws->phase = phase; + ZSTD_cwksp_assert_internal_consistency(ws); + } + return 0; +} + +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) +{ + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd); +} + +/** + * Internal function. Do not use directly. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) +{ + void* alloc; + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) { + return NULL; + } + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + if (alloc) { + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + /* We need to keep the redzone poisoned while unpoisoning the bytes that + * are actually allocated. */ + __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE); + } + } +#endif + + return alloc; +} + +/** + * Reserves and returns unaligned memory. + */ +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) +{ + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + * This memory has been initialized at least once in the past. + * This doesn't mean it has been initialized this time, and it might contain data from previous + * operations. + * The main usage is for algorithms that might need read access into uninitialized memory. + * The algorithm must maintain safety under these conditions and must make sure it doesn't + * leak any of the past data (directly or in side channels). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES); + void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + if(ptr && ptr < ws->initOnceStart) { + /* We assume the memory following the current allocation is either: + * 1. Not usable as initOnce memory (end of workspace) + * 2. Another initOnce buffer that has been allocated before (and so was previously memset) + * 3. An ASAN redzone, in which case we don't want to write on it + * For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart. + * Note that we assume here that MSAN and ASAN cannot run in the same time. */ + ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes)); + ws->initOnceStart = ptr; + } +#if ZSTD_MEMORY_SANITIZER + assert(__msan_test_shadow(ptr, bytes) == -1); +#endif + return ptr; +} + +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) +{ + void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES), + ZSTD_cwksp_alloc_aligned); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + return ptr; +} + +/** + * Aligned on 64 bytes. These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) +{ + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once; + void* alloc; + void* end; + void* top; + + /* We can only start allocating tables after we are done reserving space for objects at the + * start of the workspace */ + if(ws->phase < phase) { + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) { + return NULL; + } + } + alloc = ws->tableEnd; + end = (BYTE *)alloc + bytes; + top = ws->allocStart; + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: table alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + return alloc; +} + +/** + * Aligned on sizeof(void*). + * Note : should happen only once, at workspace first initialization + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(4, + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0); + assert(bytes % ZSTD_ALIGNOF(void*) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(3, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + ws->tableValidEnd = end; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + return alloc; +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) +{ + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * Since tableValidEnd space and initOnce space may overlap we don't poison + * the initOnce portion as it break its promise. This means that this poisoning + * check isn't always applied fully. */ + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + assert(__msan_test_shadow(ws->objectEnd, size) == -1); + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + __msan_poison(ws->objectEnd, size); + } else { + assert(ws->initOnceStart >= ws->objectEnd); + __msan_poison(ws->objectEnd, (BYTE*)ws->initOnceStart - (BYTE*)ws->objectEnd); + } + } +#endif + + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Zero the part of the allocated tables not already marked clean. + */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd)); + } + ZSTD_cwksp_mark_tables_clean(ws); +} + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the context re-use logic is sound, and that we don't + * access stuff that this compression hasn't initialized, we re-"poison" + * the workspace except for the areas in which we expect memory re-use + * without initialization (objects, valid tables area and init once + * memory). */ + { + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + size_t size = (BYTE*)ws->initOnceStart - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } + } +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ws->allocStart = ZSTD_cwksp_initialAllocStart(ws); + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) { + ws->phase = ZSTD_cwksp_alloc_aligned_init_once; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + ws->phase = ZSTD_cwksp_alloc_objects; + ws->isStatic = isStatic; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_customMalloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); + ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc); + return 0; +} + +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; + DEBUGLOG(4, "cwksp: freeing workspace"); + ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_customFree(ptr, customMem); +} + +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before it's used again). + */ +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); +} + +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} + +MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) + + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); +} + +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} + +/*-************************************* +* Functions Checking Free Space +***************************************/ + +/* ZSTD_alignmentSpaceWithinBounds() : + * Returns if the estimated space needed for a wksp is within an acceptable limit of the + * actual amount of space used. + */ +MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) { + /* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice + * the alignment bytes difference between estimation and actual usage */ + return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) && + ZSTD_cwksp_used(ws) <= estimatedSpace; +} + + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CWKSP_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.c new file mode 100644 index 000000000..0ad88ffc7 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.c @@ -0,0 +1,758 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_double_fast.h" + +static void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) { + ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i); + } + if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { + ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i); + } + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + +static void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) + hashSmall[smHash] = curr + i; + if (i == 0 || hashLarge[lgHash] == 0) + hashLarge[lgHash] = curr + i; + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm); + } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_noDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t mLength; + U32 offset; + U32 curr; + + /* how many positions to search before increasing step size */ + const size_t kStepIncr = 1 << kSearchStrength; + /* the position at which to increment the step size if no match is found */ + const BYTE* nextStep; + size_t step; /* the current step size */ + + size_t hl0; /* the long hash at ip */ + size_t hl1; /* the long hash at ip1 */ + + U32 idxl0; /* the long match index for ip */ + U32 idxl1; /* the long match index for ip1 */ + + const BYTE* matchl0; /* the long match for ip */ + const BYTE* matchs0; /* the short match for ip */ + const BYTE* matchl1; /* the long match for ip1 */ + + const BYTE* ip = istart; /* the current position */ + const BYTE* ip1; /* the next position */ + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); + + /* init */ + ip += ((ip - prefixLowest) == 0); + { + U32 const current = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); + U32 const maxRep = current - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* Outer Loop: one iteration per match found and stored */ + while (1) { + step = 1; + nextStep = ip + kStepIncr; + ip1 = ip + step; + + if (ip1 > ilimit) { + goto _cleanup; + } + + hl0 = ZSTD_hashPtr(ip, hBitsL, 8); + idxl0 = hashLong[hl0]; + matchl0 = base + idxl0; + + /* Inner Loop: one iteration per search / position */ + do { + const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 idxs0 = hashSmall[hs0]; + curr = (U32)(ip-base); + matchs0 = base + idxs0; + + hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ + + /* check noDict repcode */ + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); + + if (idxl0 > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchl0) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; + offset = (U32)(ip-matchl0); + while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } + } + + idxl1 = hashLong[hl1]; + matchl1 = base + idxl1; + + if (idxs0 > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(matchs0) == MEM_read32(ip)) { + goto _search_next_long; + } + } + + if (ip1 >= nextStep) { + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + step++; + nextStep += kStepIncr; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + matchl0 = matchl1; + #if defined(__aarch64__) + PREFETCH_L1(ip+256); + #endif + } while (ip1 <= ilimit); + +_cleanup: + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_search_next_long: + + /* check prefix long +1 match */ + if (idxl1 > prefixLowestIndex) { + if (MEM_read64(matchl1) == MEM_read64(ip1)) { + ip = ip1; + mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8; + offset = (U32)(ip-matchl1); + while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */ + goto _match_found; + } + } + + /* if no long +1 match, explore the short match we found */ + mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; + offset = (U32)(ip - matchs0); + while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */ + + /* fall-through */ + +_match_found: /* requires ip, offset, mLength */ + offset_2 = offset_1; + offset_1 = offset; + + if (step < 4) { + /* It is unsafe to write this value back to the hashtable when ip1 is + * greater than or equal to the new ip we will have after we're done + * processing this match. Rather than perform that test directly + * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler + * more predictable test. The minmatch even if we take a short match is + * 4 bytes, so as long as step, the distance between ip and ip1 + * (initially) is less than 4, we know ip1 < new ip. */ + hashLong[hl1] = (U32)(ip1 - base); + } + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams; + const U32* const dictHashLong = dms->hashTable; + const U32* const dictHashSmall = dms->chainTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase); + const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); + + /* if a dictionary is attached, it must be within window range */ + assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32); + PREFETCH_AREA(dictHashLong, hashTableBytes) + PREFETCH_AREA(dictHashSmall, chainTableBytes) + } + + /* init */ + ip += (dictAndPrefixLength == 0); + + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + U32 offset; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS]; + U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL); + int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS); + U32 const curr = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ + + /* check repcode */ + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + if (matchIndexL > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchLong) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + goto _match_found; + } + } else if (dictTagsMatchL) { + /* check dictMatchState long match */ + U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatchL = dictBase + dictMatchIndexL; + assert(dictMatchL < dictEnd); + + if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { + mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; + offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); + while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ + goto _match_found; + } } + + if (matchIndexS > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } + } else if (dictTagsMatchS) { + /* check dictMatchState short match */ + U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS; + match = dictBase + dictMatchIndexS; + matchIndexS = dictMatchIndexS + dictIndexDelta; + + if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } } + + ip += ((ip-anchor) >> kSearchStrength) + 1; +#if defined(__aarch64__) + PREFETCH_L1(ip+256); +#endif + continue; + +_search_next_long: + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3); + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = curr + 1; + + /* check prefix long +1 match */ + if (matchIndexL3 > prefixLowestIndex) { + if (MEM_read64(matchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + ip++; + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ + goto _match_found; + } + } else if (dictTagsMatchL3) { + /* check dict long +1 match */ + U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; + assert(dictMatchL3 < dictEnd); + if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; + ip++; + offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); + while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ + goto _match_found; + } } } + + /* if no long +1 match, explore the short match we found */ + if (matchIndexS < prefixLowestIndex) { + mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; + offset = (U32)(curr - matchIndexS); + while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip - match); + while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + +_match_found: + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ? + dictBase + repIndex2 - dictIndexDelta : + base + repIndex2; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + } /* while (ip < ilimit) */ + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_DFAST_FN(noDict, 4) +ZSTD_GEN_DFAST_FN(noDict, 5) +ZSTD_GEN_DFAST_FN(noDict, 6) +ZSTD_GEN_DFAST_FN(noDict, 7) + +ZSTD_GEN_DFAST_FN(dictMatchState, 4) +ZSTD_GEN_DFAST_FN(dictMatchState, 5) +ZSTD_GEN_DFAST_FN(dictMatchState, 6) +ZSTD_GEN_DFAST_FN(dictMatchState, 7) + + +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); + } +} + + +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + } +} + + +static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + U32 offset_1=rep[0], offset_2=rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ + + if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ + & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + } else { + if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; + offset = curr - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + + } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = curr + 1; + if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; + ip++; + offset = curr+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + offset = curr - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + + } else { + ip += ((ip-anchor) >> kSearchStrength) + 1; + continue; + } } + + /* move to next sequence start */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */ + & (offset_2 <= current2 - dictStartIndex)) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +ZSTD_GEN_DFAST_FN(extDict, 4) +ZSTD_GEN_DFAST_FN(extDict, 5) +ZSTD_GEN_DFAST_FN(extDict, 6) +ZSTD_GEN_DFAST_FN(extDict, 7) + +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.h new file mode 100644 index 000000000..6f0047c4b --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DOUBLE_FAST_H +#define ZSTD_DOUBLE_FAST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_DOUBLE_FAST_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.c new file mode 100644 index 000000000..5f2c6a2ed --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.c @@ -0,0 +1,960 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ +#include "zstd_fast.h" + +static void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Currently, we always use ZSTD_dtlm_full for filling CDict tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_full); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls); + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); } + + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */ + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p); + } } } } +} + +static void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_fast); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); + hashTable[hash0] = curr; + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hash] == 0) { /* not yet filled */ + hashTable[hash] = curr + p; + } } } } +} + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillHashTableForCCtx(ms, end, dtlm); + } +} + + +/** + * If you squint hard enough (and ignore repcodes), the search operation at any + * given position is broken into 4 stages: + * + * 1. Hash (map position to hash value via input read) + * 2. Lookup (map hash val to index via hashtable read) + * 3. Load (map index to value at that position via input read) + * 4. Compare + * + * Each of these steps involves a memory read at an address which is computed + * from the previous step. This means these steps must be sequenced and their + * latencies are cumulative. + * + * Rather than do 1->2->3->4 sequentially for a single position before moving + * onto the next, this implementation interleaves these operations across the + * next few positions: + * + * R = Repcode Read & Compare + * H = Hash + * T = Table Lookup + * M = Match Read & Compare + * + * Pos | Time --> + * ----+------------------- + * N | ... M + * N+1 | ... TM + * N+2 | R H T M + * N+3 | H TM + * N+4 | R H T M + * N+5 | H ... + * N+6 | R ... + * + * This is very much analogous to the pipelining of execution in a CPU. And just + * like a CPU, we have to dump the pipeline when we find a match (i.e., take a + * branch). + * + * When this happens, we throw away our current state, and do the following prep + * to re-enter the loop: + * + * Pos | Time --> + * ----+------------------- + * N | H T + * N+1 | H + * + * This is also the work we do at the beginning to enter the loop initially. + */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_fast_noDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + + const BYTE* anchor = istart; + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + U32 rep_offset1 = rep[0]; + U32 rep_offset2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + U32 mval; /* src value at match idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + + /* ip0 and ip1 are always adjacent. The targetLength skipping and + * uncompressibility acceleration is applied to every other position, + * matching the behavior of #1562. step therefore represents the gap + * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */ + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); + ip0 += (ip0 == prefixStart); + { U32 const curr = (U32)(ip0 - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); + U32 const maxRep = curr - windowLow; + if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0; + if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0; + } + + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + + do { + /* load repcode match for ip[2]*/ + const U32 rval = MEM_read32(ip2 - rep_offset1); + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) { + ip0 = ip2; + match0 = ip0 - rep_offset1; + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + + /* First write next hash table entry; we've already calculated it. + * This write is known to be safe because the ip1 is before the + * repcode (ip2). */ + hashTable[hash1] = (U32)(ip1 - base); + + goto _match; + } + + /* load match for ip[0] */ + if (idx >= prefixStartIndex) { + mval = MEM_read32(base + idx); + } else { + mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */ + } + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + + /* First write next hash table entry; we've already calculated it. + * This write is known to be safe because the ip1 == ip0 + 1, so + * we know we will resume searching after ip1 */ + hashTable[hash1] = (U32)(ip1 - base); + + goto _offset; + } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* load match for ip[0] */ + if (idx >= prefixStartIndex) { + mval = MEM_read32(base + idx); + } else { + mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */ + } + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + + /* first write next hash table entry; we've already calculated it */ + if (step <= 4) { + /* We need to avoid writing an index into the hash table >= the + * position at which we will pick up our searching after we've + * taken this match. + * + * The minimum possible match has length 4, so the earliest ip0 + * can be after we take this match will be the current ip0 + 4. + * ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely + * write this position. + */ + hashTable[hash1] = (U32)(ip1 - base); + } + + goto _offset; + } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* When the repcodes are outside of the prefix, we set them to zero before the loop. + * When the offsets are still zero, we need to restore them after the block to have a correct + * repcode history. If only one offset was invalid, it is easy. The tricky case is when both + * offsets were invalid. We need to figure out which offset to refill with. + * - If both offsets are zero they are in the same order. + * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`. + * - If only one is zero, we need to decide which offset to restore. + * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1. + * - It is impossible for rep_offset2 to be non-zero. + * + * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then + * set rep[0] = rep_offset1 and rep[1] = offsetSaved1. + */ + offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1; + rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx */ + + /* Compute the offset code. */ + match0 = base + idx; + rep_offset2 = rep_offset1; + rep_offset1 = (U32)(ip0-match0); + offcode = OFFSET_TO_OFFBASE(rep_offset1); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } + +_match: /* Requires: ip0, match0, offcode */ + + /* Count the forward length. */ + mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */ + while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4; + { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } } } + + goto _start; +} + +#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \ + static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \ + } + +ZSTD_GEN_FAST_FN(noDict, 4, 1) +ZSTD_GEN_FAST_FN(noDict, 5, 1) +ZSTD_GEN_FAST_FN(noDict, 6, 1) +ZSTD_GEN_FAST_FN(noDict, 7, 1) + +ZSTD_GEN_FAST_FN(noDict, 4, 0) +ZSTD_GEN_FAST_FN(noDict, 5, 0) +ZSTD_GEN_FAST_FN(noDict, 6, 0) +ZSTD_GEN_FAST_FN(noDict, 7, 0) + +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + if (ms->cParams.targetLength > 1) { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize); + } + } else { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize); + } + + } +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_fast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip0 = istart; + const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */ + const BYTE* anchor = istart; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; + const U32* const dictHashTable = dms->hashTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); + const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart); + const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + + /* if a dictionary is still attached, it necessarily means that + * it is within window size. So we just check it. */ + const U32 maxDistance = 1U << cParams->windowLog; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + assert(endIndex - prefixStartIndex <= maxDistance); + (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + + (void)hasStep; /* not currently specialized on whether it's accelerated */ + + /* ensure there will be no underflow + * when translating a dict index into a local index */ + assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); + + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + PREFETCH_AREA(dictHashTable, hashTableBytes) + } + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); + ip0 += (dictAndPrefixLength == 0); + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Outer search loop */ + assert(stepSize >= 1); + while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */ + size_t mLength; + size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls); + + size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls); + U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0); + + U32 matchIndex = hashTable[hash0]; + U32 curr = (U32)(ip0 - base); + size_t step = stepSize; + const size_t kStepIncr = 1 << kSearchStrength; + const BYTE* nextStep = ip0 + kStepIncr; + + /* Inner search loop */ + while (1) { + const BYTE* match = base + matchIndex; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixStartIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls); + size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls); + hashTable[hash0] = curr; /* update hash table */ + + if (((U32) ((prefixStartIndex - 1) - repIndex) >= + 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ + && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip0++; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + break; + } + + if (dictTagsMatch) { + /* Found a possible dict match */ + const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex > dictStartIndex && + MEM_read32(dictMatch) == MEM_read32(ip0)) { + /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */ + if (matchIndex <= prefixStartIndex) { + U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta); + mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4; + while (((ip0 > anchor) & (dictMatch > dictStart)) + && (ip0[-1] == dictMatch[-1])) { + ip0--; + dictMatch--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + } + } + + if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) { + /* found a regular match */ + U32 const offset = (U32) (ip0 - match); + mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4; + while (((ip0 > anchor) & (match > prefixStart)) + && (ip0[-1] == match[-1])) { + ip0--; + match--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + + /* Prepare for next iteration */ + dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS]; + dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1); + matchIndex = hashTable[hash1]; + + if (ip1 >= nextStep) { + step++; + nextStep += kStepIncr; + } + ip0 = ip1; + ip1 = ip1 + step; + if (ip1 > ilimit) goto _cleanup; + + curr = (U32)(ip0 - base); + hash0 = hash1; + } /* end inner search loop */ + + /* match found */ + assert(mLength); + ip0 += mLength; + anchor = ip0; + + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+curr+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + /* check immediate repcode */ + while (ip0 <= ilimit) { + U32 const current2 = (U32)(ip0-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? + dictBase - dictIndexDelta + repIndex2 : + base + repIndex2; + if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip0))) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2; + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } + } + + /* Prepare for next iteration */ + assert(ip0 == anchor); + ip1 = ip0 + stepSize; + } + +_cleanup: + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +ZSTD_GEN_FAST_FN(dictMatchState, 4, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 5, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 6, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 7, 0) + +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState != NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); + } +} + + +static size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const BYTE* const dictStart = dictBase + dictStartIndex; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + const BYTE* idxBase; /* base pointer for idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */ + + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + + (void)hasStep; /* not currently specialized on whether it's accelerated */ + + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); + + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); + + { U32 const curr = (U32)(ip0 - base); + U32 const maxRep = curr - dictStartIndex; + if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + do { + { /* load repcode match for ip[2] */ + U32 const current2 = (U32)(ip2 - base); + U32 const repIndex = current2 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + U32 rval; + if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */ + & (offset_1 > 0) ) { + rval = MEM_read32(repBase + repIndex); + } else { + rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */ + } + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if (MEM_read32(ip2) == rval) { + ip0 = ip2; + match0 = repBase + repIndex; + matchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + assert((match0 != prefixStart) & (match0 != dictStart)); + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + goto _match; + } } + + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx, idxBase */ + + /* Compute the offset code. */ + { U32 const offset = current0 - idx; + const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart; + matchEnd = idx < prefixStartIndex ? dictEnd : iend; + match0 = idxBase + idx; + offset_2 = offset_1; + offset_1 = offset; + offcode = OFFSET_TO_OFFBASE(offset); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } } + +_match: /* Requires: ip0, match0, offcode, matchEnd */ + + /* Count the forward length. */ + assert(matchEnd != 0); + mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* write next hash table entry */ + if (ip1 < ip0) { + hashTable[hash1] = (U32)(ip1 - base); + } + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + while (ip0 <= ilimit) { + U32 const repIndex2 = (U32)(ip0-base) - offset_2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0)) /* intentional underflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } } + + goto _start; +} + +ZSTD_GEN_FAST_FN(extDict, 4, 0) +ZSTD_GEN_FAST_FN(extDict, 5, 0) +ZSTD_GEN_FAST_FN(extDict, 6, 0) +ZSTD_GEN_FAST_FN(extDict, 7, 0) + +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize); + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.h new file mode 100644 index 000000000..9e4236b47 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_FAST_H +#define ZSTD_FAST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c new file mode 100644 index 000000000..5ba88e867 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c @@ -0,0 +1,2157 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_lazy.h" +#include "../common/bits.h" /* ZSTD_countTrailingZeros64 */ + +#define kLazySkippingStep 8 + + +/*-************************************* +* Binary Tree search +***************************************/ + +static void +ZSTD_updateDUBT(ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iend, + U32 mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + if (idx != target) + DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", + idx, target, ms->window.dictLimit); + assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ + (void)iend; + + assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ + for ( ; idx < target ; idx++) { + size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ + U32 const matchIndex = hashTable[h]; + + U32* const nextCandidatePtr = bt + 2*(idx&btMask); + U32* const sortMarkPtr = nextCandidatePtr + 1; + + DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx); + hashTable[h] = idx; /* Update Hash Table */ + *nextCandidatePtr = matchIndex; /* update BT like a chain */ + *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; + } + ms->nextToUpdate = target; +} + + +/** ZSTD_insertDUBT1() : + * sort one already inserted but unsorted position + * assumption : curr >= btlow == (curr - btmask) + * doesn't fail */ +static void +ZSTD_insertDUBT1(const ZSTD_matchState_t* ms, + U32 curr, const BYTE* inputEnd, + U32 nbCompares, U32 btLow, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; + const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ + U32 dummy32; /* to be nullified at the end */ + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; + + + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", + curr, dictLimit, windowLow); + assert(curr >= btLow); + assert(ip < iend); /* condition for ZSTD_count */ + + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + /* note : all candidates are now supposed sorted, + * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK + * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ + + if ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ + || (curr < dictLimit) /* both in extDict */) { + const BYTE* const mBase = ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit)) ? + base : dictBase; + assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ + || (curr < dictLimit) ); + match = mBase + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* preparation for next read of match[matchLength] */ + } + + DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", + curr, matchIndex, (U32)matchLength); + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u", + matchIndex, btLow, nextPtr[1]); + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u", + matchIndex, btLow, nextPtr[0]); + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; +} + + +static size_t +ZSTD_DUBT_findBetterDictMatch ( + const ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + size_t bestLength, + U32 nbCompares, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_matchState_t * const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; + const U32 * const dictHashTable = dms->hashTable; + U32 const hashLog = dmsCParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 dictMatchIndex = dictHashTable[h]; + + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + U32 const curr = (U32)(ip-base); + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictEnd = dms->window.nextSrc; + U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); + U32 const dictLowLimit = dms->window.lowLimit; + U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit; + + U32* const dictBt = dms->chainTable; + U32 const btLog = dmsCParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; + + size_t commonLengthSmaller=0, commonLengthLarger=0; + + (void)dictMode; + assert(dictMode == ZSTD_dictMatchState); + + for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) { + U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dictBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (dictMatchIndex+matchLength >= dictHighLimit) + match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + U32 matchIndex = dictMatchIndex + dictIndexDelta; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { + DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + } + if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + } + return bestLength; + +} + + +static size_t +ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offBasePtr, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const unsortLimit = MAX(btLow, windowLow); + + U32* nextCandidate = bt + 2*(matchIndex&btMask); + U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; + U32 nbCompares = 1U << cParams->searchLog; + U32 nbCandidates = nbCompares; + U32 previousCandidate = 0; + + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr); + assert(ip <= iend-8); /* required for h calculation */ + assert(dictMode != ZSTD_dedicatedDictSearch); + + /* reach end of unsorted candidates list */ + while ( (matchIndex > unsortLimit) + && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK) + && (nbCandidates > 1) ) { + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", + matchIndex); + *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ + previousCandidate = matchIndex; + matchIndex = *nextCandidate; + nextCandidate = bt + 2*(matchIndex&btMask); + unsortedMark = bt + 2*(matchIndex&btMask) + 1; + nbCandidates --; + } + + /* nullify last candidate if it's still unsorted + * simplification, detrimental to compression ratio, beneficial for speed */ + if ( (matchIndex > unsortLimit) + && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) { + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", + matchIndex); + *nextCandidate = *unsortedMark = 0; + } + + /* batch sort stacked candidates */ + matchIndex = previousCandidate; + while (matchIndex) { /* will end on matchIndex == 0 */ + U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; + U32 const nextCandidateIdx = *nextCandidateIdxPtr; + ZSTD_insertDUBT1(ms, matchIndex, iend, + nbCandidates, unsortLimit, dictMode); + matchIndex = nextCandidateIdx; + nbCandidates++; + } + + /* find longest match */ + { size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr + 8 + 1; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + matchIndex = hashTable[h]; + hashTable[h] = curr; /* Update Hash Table */ + + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) ) + bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + if (dictMode == ZSTD_dictMatchState) { + nbCompares = 0; /* in addition to avoiding checking any + * further in this loop, make sure we + * skip checking in the dictionary. */ + } + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + bestLength = ZSTD_DUBT_findBetterDictMatch( + ms, ip, iend, + offBasePtr, bestLength, nbCompares, + mls, dictMode); + } + + assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offBasePtr, mIndex); + } + return bestLength; + } +} + + +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offBasePtr, + const U32 mls /* template */, + const ZSTD_dictMode_e dictMode) +{ + DEBUGLOG(7, "ZSTD_BtFindBestMatch"); + if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateDUBT(ms, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode); +} + +/*********************************** +* Dedicated dict search +***********************************/ + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32* const hashTable = ms->hashTable; + U32* const chainTable = ms->chainTable; + U32 const chainSize = 1 << ms->cParams.chainLog; + U32 idx = ms->nextToUpdate; + U32 const minChain = chainSize < target - idx ? target - chainSize : idx; + U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const cacheSize = bucketSize - 1; + U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; + U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts; + + /* We know the hashtable is oversized by a factor of `bucketSize`. + * We are going to temporarily pretend `bucketSize == 1`, keeping only a + * single entry. We will use the rest of the space to construct a temporary + * chaintable. + */ + U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + U32* const tmpHashTable = hashTable; + U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); + U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; + U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; + U32 hashIdx; + + assert(ms->cParams.chainLog <= 24); + assert(ms->cParams.hashLog > ms->cParams.chainLog); + assert(idx != 0); + assert(tmpMinChain <= minChain); + + /* fill conventional hash table and conventional chain table */ + for ( ; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch); + if (idx >= tmpMinChain) { + tmpChainTable[idx - tmpMinChain] = hashTable[h]; + } + tmpHashTable[h] = idx; + } + + /* sort chains into ddss chain table */ + { + U32 chainPos = 0; + for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) { + U32 count; + U32 countBeyondMinChain = 0; + U32 i = tmpHashTable[hashIdx]; + for (count = 0; i >= tmpMinChain && count < cacheSize; count++) { + /* skip through the chain to the first position that won't be + * in the hash cache bucket */ + if (i < minChain) { + countBeyondMinChain++; + } + i = tmpChainTable[i - tmpMinChain]; + } + if (count == cacheSize) { + for (count = 0; count < chainLimit;) { + if (i < minChain) { + if (!i || ++countBeyondMinChain > cacheSize) { + /* only allow pulling `cacheSize` number of entries + * into the cache or chainTable beyond `minChain`, + * to replace the entries pulled out of the + * chainTable into the cache. This lets us reach + * back further without increasing the total number + * of entries in the chainTable, guaranteeing the + * DDSS chain table will fit into the space + * allocated for the regular one. */ + break; + } + } + chainTable[chainPos++] = i; + count++; + if (i < tmpMinChain) { + break; + } + i = tmpChainTable[i - tmpMinChain]; + } + } else { + count = 0; + } + if (count) { + tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count; + } else { + tmpHashTable[hashIdx] = 0; + } + } + assert(chainPos <= chainSize); /* I believe this is guaranteed... */ + } + + /* move chain pointers into the last entry of each hash bucket */ + for (hashIdx = (1 << hashLog); hashIdx; ) { + U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const chainPackedPointer = tmpHashTable[hashIdx]; + U32 i; + for (i = 0; i < cacheSize; i++) { + hashTable[bucketIdx + i] = 0; + } + hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer; + } + + /* fill the buckets of the hash table */ + for (idx = ms->nextToUpdate; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch) + << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 i; + /* Shift hash cache down 1. */ + for (i = cacheSize - 1; i; i--) + hashTable[h + i] = hashTable[h + i - 1]; + hashTable[h] = idx; + } + + ms->nextToUpdate = target; +} + +/* Returns the longest match length found in the dedicated dict search structure. + * If none are longer than the argument ml, then ml will be returned. + */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts, + const ZSTD_matchState_t* const dms, + const BYTE* const ip, const BYTE* const iLimit, + const BYTE* const prefixStart, const U32 curr, + const U32 dictLimit, const size_t ddsIdx) { + const U32 ddsLowestIndex = dms->window.dictLimit; + const BYTE* const ddsBase = dms->window.base; + const BYTE* const ddsEnd = dms->window.nextSrc; + const U32 ddsSize = (U32)(ddsEnd - ddsBase); + const U32 ddsIndexDelta = dictLimit - ddsSize; + const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); + const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; + U32 ddsAttempt; + U32 matchIndex; + + for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { + PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 const chainIndex = chainPackedPointer >> 8; + + PREFETCH_L1(&dms->chainTable[chainIndex]); + } + + for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; + match = ddsBase + matchIndex; + + if (!matchIndex) { + return ml; + } + + /* guaranteed by table construction */ + (void)ddsLowestIndex; + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) { + /* best possible, avoids read overflow on next attempt */ + return ml; + } + } + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 chainIndex = chainPackedPointer >> 8; + U32 const chainLength = chainPackedPointer & 0xFF; + U32 const chainAttempts = nbAttempts - ddsAttempt; + U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; + U32 chainAttempt; + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { + PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); + } + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->chainTable[chainIndex]; + match = ddsBase + matchIndex; + + /* guaranteed by table construction */ + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + return ml; +} + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_matchState_t* ms, + const ZSTD_compressionParameters* const cParams, + const BYTE* ip, U32 const mls, U32 const lazySkipping) +{ + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + /* Stop inserting every position when in the lazy skipping mode. */ + if (lazySkipping) + break; + } + + ms->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { + const ZSTD_compressionParameters* const cParams = &ms->cParams; + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0); +} + +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_HcFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const chainTable = ms->chainTable; + const U32 chainSize = (1 << cParams->chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 minChain = curr > chainSize ? curr - chainSize : 0; + U32 nbAttempts = 1U << cParams->searchLog; + size_t ml=4-1; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch + ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch + ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + + U32 matchIndex; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32* entry = &dms->hashTable[ddsIdx]; + PREFETCH_L1(entry); + } + + /* HC4 match finder */ + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping); + + for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + const U32* const dmsChainTable = dms->chainTable; + const U32 dmsChainSize = (1 << dms->cParams.chainLog); + const U32 dmsChainMask = dmsChainSize - 1; + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0; + + matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; + + for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= dmsMinChain) break; + + matchIndex = dmsChainTable[matchIndex & dmsChainMask]; + } + } + + return ml; +} + +/* ********************************* +* (SIMD) Row-based matchfinder +***********************************/ +/* Constants for row-based hash */ +#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1) +#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */ + +#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1) + +typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */ + +/* ZSTD_VecMask_next(): + * Starting from the LSB, returns the idx of the next non-zero bit. + * Basically counting the nb of trailing zeroes. + */ +MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) { + return ZSTD_countTrailingZeros64(val); +} + +/* ZSTD_row_nextIndex(): + * Returns the next index to insert at within a tagTable row, and updates the "head" + * value to reflect the update. Essentially cycles backwards from [1, {entries per row}) + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) { + U32 next = (*tagRow-1) & rowMask; + next += (next == 0) ? rowMask : 0; /* skip first position */ + *tagRow = (BYTE)next; + return next; +} + +/* ZSTD_isAligned(): + * Checks that a pointer is aligned to "align" bytes which must be a power of 2. + */ +MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) { + assert((align & (align - 1)) == 0); + return (((size_t)ptr) & (align - 1)) == 0; +} + +/* ZSTD_row_prefetch(): + * Performs prefetching for the hashTable and tagTable at a given row. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) { + PREFETCH_L1(hashTable + relRow); + if (rowLog >= 5) { + PREFETCH_L1(hashTable + relRow + 16); + /* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */ + } + PREFETCH_L1(tagTable + relRow); + if (rowLog == 6) { + PREFETCH_L1(tagTable + relRow + 32); + } + assert(rowLog == 4 || rowLog == 5 || rowLog == 6); + assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */ + assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */ +} + +/* ZSTD_row_fillHashCache(): + * Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries, + * but not beyond iLimit. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base, + U32 const rowLog, U32 const mls, + U32 idx, const BYTE* const iLimit) +{ + U32 const* const hashTable = ms->hashTable; + BYTE const* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1); + U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch); + + for (; idx < lim; ++idx) { + U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash; + } + + DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1], + ms->hashCache[2], ms->hashCache[3], ms->hashCache[4], + ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]); +} + +/* ZSTD_row_nextCachedHash(): + * Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at + * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable. + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable, + BYTE const* tagTable, BYTE const* base, + U32 idx, U32 const hashLog, + U32 const rowLog, U32 const mls, + U64 const hashSalt) +{ + U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK]; + cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash; + return hash; + } +} + +/* ZSTD_row_update_internalImpl(): + * Updates the hash table with positions starting from updateStartIdx until updateEndIdx. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms, + U32 updateStartIdx, U32 const updateEndIdx, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) +{ + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + const BYTE* const base = ms->window.base; + + DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx); + for (; updateStartIdx < updateEndIdx; ++updateStartIdx) { + U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt) + : (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32* const row = hashTable + relRow; + BYTE* tagRow = tagTable + relRow; + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + + assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt)); + tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK; + row[pos] = updateStartIdx; + } +} + +/* ZSTD_row_update_internal(): + * Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate. + * Skips sections of long matches as is necessary. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) +{ + U32 idx = ms->nextToUpdate; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + const U32 kSkipThreshold = 384; + const U32 kMaxMatchStartPositionsToUpdate = 96; + const U32 kMaxMatchEndPositionsToUpdate = 32; + + if (useCache) { + /* Only skip positions when using hash cache, i.e. + * if we are loading a dict, don't skip anything. + * If we decide to skip, then we only update a set number + * of positions at the beginning and end of the match. + */ + if (UNLIKELY(target - idx > kSkipThreshold)) { + U32 const bound = idx + kMaxMatchStartPositionsToUpdate; + ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache); + idx = target - kMaxMatchEndPositionsToUpdate; + ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1); + } + } + assert(target >= idx); + ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache); + ms->nextToUpdate = target; +} + +/* ZSTD_row_update(): + * External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary + * processing. + */ +void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) { + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + const U32 rowMask = (1u << rowLog) - 1; + const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */); + + DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog); + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */); +} + +/* Returns the mask width of bits group of which will be set to 1. Given not all + * architectures have easy movemask instruction, this helps to iterate over + * groups of bits easier and faster. + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_row_matchMaskGroupWidth(const U32 rowEntries) +{ + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES); + (void)rowEntries; +#if defined(ZSTD_ARCH_ARM_NEON) + /* NEON path only works for little endian */ + if (!MEM_isLittleEndian()) { + return 1; + } + if (rowEntries == 16) { + return 4; + } + if (rowEntries == 32) { + return 2; + } + if (rowEntries == 64) { + return 1; + } +#endif + return 1; +} + +#if defined(ZSTD_ARCH_X86_SSE2) +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head) +{ + const __m128i comparisonMask = _mm_set1_epi8((char)tag); + int matches[4] = {0}; + int i; + assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4); + for (i=0; i> chunkSize; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= (chunk * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } else { /* big endian: reverse bits during extraction */ + const size_t msb = xFF ^ (xFF >> 1); + const size_t extractMagic = (msb / 0x1FF) | msb; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= ((chunk >> 7) * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } + matches = ~matches; + if (rowEntries == 16) { + return ZSTD_rotateRight_U16((U16)matches, headGrouped); + } else if (rowEntries == 32) { + return ZSTD_rotateRight_U32((U32)matches, headGrouped); + } else { + return ZSTD_rotateRight_U64((U64)matches, headGrouped); + } + } +#endif +} + +/* The high-level approach of the SIMD row based match finder is as follows: + * - Figure out where to insert the new entry: + * - Generate a hash from a byte along with an additional 1-byte "short hash". The additional byte is our "tag" + * - The hashTable is effectively split into groups or "rows" of 16 or 32 entries of U32, and the hash determines + * which row to insert into. + * - Determine the correct position within the row to insert the entry into. Each row of 16 or 32 can + * be considered as a circular buffer with a "head" index that resides in the tagTable. + * - Also insert the "tag" into the equivalent row and position in the tagTable. + * - Note: The tagTable has 17 or 33 1-byte entries per row, due to 16 or 32 tags, and 1 "head" entry. + * The 17 or 33 entry rows are spaced out to occur every 32 or 64 bytes, respectively, + * for alignment/performance reasons, leaving some bytes unused. + * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte "short hash" and + * generate a bitfield that we can cycle through to check the collisions in the hash table. + * - Pick the longest match. + */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_RowFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode, + const U32 rowLog) +{ + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32* const hashCache = ms->hashCache; + const U32 hashLog = ms->rowHashLog; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 rowEntries = (1U << rowLog); + const U32 rowMask = rowEntries - 1; + const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */ + const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries); + const U64 hashSalt = ms->hashSalt; + U32 nbAttempts = 1U << cappedSearchLog; + size_t ml=4-1; + U32 hash; + + /* DMS/DDS variables that may be referenced laster */ + const ZSTD_matchState_t* const dms = ms->dictMatchState; + + /* Initialize the following variables to satisfy static analyzer */ + size_t ddsIdx = 0; + U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */ + U32 dmsTag = 0; + U32* dmsRow = NULL; + BYTE* dmsTagRow = NULL; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + { /* Prefetch DDS hashtable entry */ + ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG; + PREFETCH_L1(&dms->hashTable[ddsIdx]); + } + ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0; + } + + if (dictMode == ZSTD_dictMatchState) { + /* Prefetch DMS rows */ + U32* const dmsHashTable = dms->hashTable; + BYTE* const dmsTagTable = dms->tagTable; + U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK; + dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow); + dmsRow = dmsHashTable + dmsRelRow; + ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog); + } + + /* Update the hashTable and tagTable up to (but not including) ip */ + if (!ms->lazySkipping) { + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */); + hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt); + } else { + /* Stop inserting every position when in the lazy skipping mode. + * The hash cache is also not kept up to date in this mode. + */ + hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + ms->nextToUpdate = curr; + } + ms->hashSaltEntropy += hash; /* collect salt entropy */ + + { /* Get the hash for ip, compute the appropriate row */ + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK; + U32* const row = hashTable + relRow; + BYTE* tagRow = (BYTE*)(tagTable + relRow); + U32 const headGrouped = (*tagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries); + + /* Cycle through the matches and prefetch */ + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = row[matchPos]; + if(matchPos == 0) continue; + assert(numMatches < rowEntries); + if (matchIndex < lowLimit) + break; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + PREFETCH_L1(base + matchIndex); + } else { + PREFETCH_L1(dictBase + matchIndex); + } + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } + + /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop + in ZSTD_row_update_internal() at the next search. */ + { + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + tagRow[pos] = (BYTE)tag; + row[pos] = ms->nextToUpdate++; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex < curr); + assert(matchIndex >= lowLimit); + + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* Save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + /* TODO: Measure and potentially add prefetching to DMS */ + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + + { U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries); + + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = dmsRow[matchPos]; + if(matchPos == 0) continue; + if (matchIndex < dmsLowestIndex) + break; + PREFETCH_L1(dmsBase + matchIndex); + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex >= dmsLowestIndex); + assert(matchIndex < curr); + + { const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + } + + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; + } + } + } + } + return ml; +} + + +/** + * Generate search functions templated on (dictMode, mls, rowLog). + * These functions are outlined for code size & compilation time. + * ZSTD_searchMax() dispatches to the correct implementation function. + * + * TODO: The start of the search function involves loading and calculating a + * bunch of constants from the ZSTD_matchState_t. These computations could be + * done in an initialization function, and saved somewhere in the match state. + * Then we could pass a pointer to the saved state instead of the match state, + * and avoid duplicate computations. + * + * TODO: Move the match re-winding into searchMax. This improves compression + * ratio, and unlocks further simplifications with the next TODO. + * + * TODO: Try moving the repcode search into searchMax. After the re-winding + * and repcode search are in searchMax, there is no more logic in the match + * finder loop that requires knowledge about the dictMode. So we should be + * able to avoid force inlining it, and we can join the extDict loop with + * the single segment loop. It should go in searchMax instead of its own + * function to avoid having multiple virtual function calls per search. + */ + +#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls +#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls +#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog + +#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE + +#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offBasePtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \ + return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \ + } \ + +#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \ + X(dictMode, mls, 4) \ + X(dictMode, mls, 5) \ + X(dictMode, mls, 6) + +#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6) + +#define ZSTD_FOR_EACH_MLS(X, dictMode) \ + X(dictMode, 4) \ + X(dictMode, 5) \ + X(dictMode, 6) + +#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ + X(__VA_ARGS__, noDict) \ + X(__VA_ARGS__, extDict) \ + X(__VA_ARGS__, dictMatchState) \ + X(__VA_ARGS__, dedicatedDictSearch) + +/* Generate row search fns for each combination of (dictMode, mls, rowLog) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN) +/* Generate binary Tree search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN) +/* Generate hash chain search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN) + +typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e; + +#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + case rowLog: \ + return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr); + +#define ZSTD_SWITCH_MLS(X, dictMode) \ + switch (mls) { \ + ZSTD_FOR_EACH_MLS(X, dictMode) \ + } + +#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \ + case mls: \ + switch (rowLog) { \ + ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \ + } \ + ZSTD_UNREACHABLE; \ + break; + +#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \ + switch (searchMethod) { \ + case search_hashChain: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \ + break; \ + case search_binaryTree: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \ + break; \ + case search_rowHash: \ + ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \ + break; \ + } \ + ZSTD_UNREACHABLE; + +/** + * Searches for the longest match at @p ip. + * Dispatches to the correct implementation function based on the + * (searchMethod, dictMode, mls, rowLog). We use switch statements + * here instead of using an indirect function call through a function + * pointer because after Spectre and Meltdown mitigations, indirect + * function calls can be very costly, especially in the kernel. + * + * NOTE: dictMode and searchMethod should be templated, so those switch + * statements should be optimized out. Only the mls & rowLog switches + * should be left. + * + * @param ms The match state. + * @param ip The position to search at. + * @param iend The end of the input data. + * @param[out] offsetPtr Stores the match offset into this pointer. + * @param mls The minimum search length, in the range [4, 6]. + * @param rowLog The row log (if applicable), in the range [4, 6]. + * @param searchMethod The search method to use (templated). + * @param dictMode The dictMode (templated). + * + * @returns The length of the longest match found, or < mls if no match is found. + * If a match is found its offset is stored in @p offsetPtr. + */ +FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax( + ZSTD_matchState_t* ms, + const BYTE* ip, + const BYTE* iend, + size_t* offsetPtr, + U32 const mls, + U32 const rowLog, + searchMethod_e const searchMethod, + ZSTD_dictMode_e const dictMode) +{ + if (dictMode == ZSTD_noDict) { + ZSTD_SWITCH_SEARCH_METHOD(noDict) + } else if (dictMode == ZSTD_extDict) { + ZSTD_SWITCH_SEARCH_METHOD(extDict) + } else if (dictMode == ZSTD_dictMatchState) { + ZSTD_SWITCH_SEARCH_METHOD(dictMatchState) + } else if (dictMode == ZSTD_dedicatedDictSearch) { + ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch) + } + ZSTD_UNREACHABLE; + return 0; +} + +/* ******************************* +* Common parser - lazy strategy +*********************************/ + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_lazy_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth, + ZSTD_dictMode_e const dictMode) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + + U32 offset_1 = rep[0], offset_2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const int isDMS = dictMode == ZSTD_dictMatchState; + const int isDDS = dictMode == ZSTD_dedicatedDictSearch; + const int isDxS = isDMS || isDDS; + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; + const BYTE* const dictBase = isDxS ? dms->window.base : NULL; + const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = isDxS ? + prefixLowestIndex - (U32)(dictEnd - dictBase) : + 0; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod); + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); + U32 const maxRep = curr - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + if (isDxS) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offBase = REPCODE1_TO_OFFBASE; + const BYTE* start=ip+1; + DEBUGLOG(7, "search baseline (depth 0)"); + + /* check repCode */ + if (isDxS) { + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch) + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + if (depth==0) goto _storeSequence; + } + } + if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + if (depth==0) goto _storeSequence; + } + + /* first search (depth 0) */ + { size_t offbaseFound = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offBase = offbaseFound; + } + + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */; + ip += step; + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + } + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + } + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* NOTE: + * Pay attention that `start[-value]` can lead to strange undefined behavior + * notably if `value` is unsigned, resulting in a large positive `-value`. + */ + /* catch up */ + if (OFFBASE_IS_OFFSET(offBase)) { + if (dictMode == ZSTD_noDict) { + while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest)) + && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */ + { start--; matchLength++; } + } + if (isDxS) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); + const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; + const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + } + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); + } + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); + anchor = ip = start + matchLength; + } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } + + /* check immediate repcode */ + if (isDxS) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex = current2 - offset_2; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase - dictIndexDelta + repIndex : + base + repIndex; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while ( ((ip <= ilimit) & (offset_2>0)) + && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { + /* store sequence */ + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); +} + + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); +} + +/* Row-based matchfinder */ +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState); +} + + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch); +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_lazy_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + + U32 offset_1 = rep[0], offset_2 = rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod); + + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + + /* init */ + ip += (ip == prefixStart); + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offBase = REPCODE1_TO_OFFBASE; + const BYTE* start=ip+1; + U32 curr = (U32)(ip-base); + + /* check repCode */ + { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog); + const U32 repIndex = (U32)(curr+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ + & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offBase = ofbCandidate; + } + + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength); + ip += step + 1; /* jump faster over incompressible sections */ + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; + } } + + /* search match, depth 1 */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; + } } + + /* search match, depth 2 */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (OFFBASE_IS_OFFSET(offBase)) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); + anchor = ip = start + matchLength; + } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repCurrent = (U32)(ip-base); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog); + const U32 repIndex = repCurrent - offset_2; + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); +} + +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); +} + +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h new file mode 100644 index 000000000..3bde67331 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LAZY_H +#define ZSTD_LAZY_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +/** + * Dedicated Dictionary Search Structure bucket log. In the + * ZSTD_dedicatedDictSearch mode, the hashTable has + * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just + * one. + */ +#define ZSTD_LAZY_DDSS_BUCKET_LOG 2 + +#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */ + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); +void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip); + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip); + +void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_LAZY_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c new file mode 100644 index 000000000..3d74ff19e --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c @@ -0,0 +1,724 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_ldm.h" + +#include "../common/debug.h" +#include "../common/xxhash.h" +#include "zstd_fast.h" /* ZSTD_fillHashTable() */ +#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ +#include "zstd_ldm_geartab.h" + +#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_MIN_MATCH_LENGTH 64 +#define LDM_HASH_RLOG 7 + +typedef struct { + U64 rolling; + U64 stopMask; +} ldmRollingHashState_t; + +/** ZSTD_ldm_gear_init(): + * + * Initializes the rolling hash state such that it will honor the + * settings in params. */ +static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params) +{ + unsigned maxBitsInMask = MIN(params->minMatchLength, 64); + unsigned hashRateLog = params->hashRateLog; + + state->rolling = ~(U32)0; + + /* The choice of the splitting criterion is subject to two conditions: + * 1. it has to trigger on average every 2^(hashRateLog) bytes; + * 2. ideally, it has to depend on a window of minMatchLength bytes. + * + * In the gear hash algorithm, bit n depends on the last n bytes; + * so in order to obtain a good quality splitting criterion it is + * preferable to use bits with high weight. + * + * To match condition 1 we use a mask with hashRateLog bits set + * and, because of the previous remark, we make sure these bits + * have the highest possible weight while still respecting + * condition 2. + */ + if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) { + state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog); + } else { + /* In this degenerate case we simply honor the hash rate. */ + state->stopMask = ((U64)1 << hashRateLog) - 1; + } +} + +/** ZSTD_ldm_gear_reset() + * Feeds [data, data + minMatchLength) into the hash without registering any + * splits. This effectively resets the hash state. This is used when skipping + * over data, either at the beginning of a block, or skipping sections. + */ +static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state, + BYTE const* data, size_t minMatchLength) +{ + U64 hash = state->rolling; + size_t n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + } while (0) + while (n + 3 < minMatchLength) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < minMatchLength) { + GEAR_ITER_ONCE(); + } +#undef GEAR_ITER_ONCE +} + +/** ZSTD_ldm_gear_feed(): + * + * Registers in the splits array all the split points found in the first + * size bytes following the data pointer. This function terminates when + * either all the data has been processed or LDM_BATCH_SIZE splits are + * present in the splits array. + * + * Precondition: The splits array must not be full. + * Returns: The number of bytes processed. */ +static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state, + BYTE const* data, size_t size, + size_t* splits, unsigned* numSplits) +{ + size_t n; + U64 hash, mask; + + hash = state->rolling; + mask = state->stopMask; + n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + if (UNLIKELY((hash & mask) == 0)) { \ + splits[*numSplits] = n; \ + *numSplits += 1; \ + if (*numSplits == LDM_BATCH_SIZE) \ + goto done; \ + } \ + } while (0) + + while (n + 3 < size) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < size) { + GEAR_ITER_ONCE(); + } + +#undef GEAR_ITER_ONCE + +done: + state->rolling = hash; + return n; +} + +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams) +{ + params->windowLog = cParams->windowLog; + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); + if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (params->hashLog == 0) { + params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + } + if (params->hashRateLog == 0) { + params->hashRateLog = params->windowLog < params->hashLog + ? 0 + : params->windowLog - params->hashLog; + } + params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); +} + +size_t ZSTD_ldm_getTableSize(ldmParams_t params) +{ + size_t const ldmHSize = ((size_t)1) << params.hashLog; + size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); + return params.enableLdm == ZSTD_ps_enable ? totalSize : 0; +} + +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) +{ + return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0; +} + +/** ZSTD_ldm_getBucket() : + * Returns a pointer to the start of the bucket associated with hash. */ +static ldmEntry_t* ZSTD_ldm_getBucket( + ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) +{ + return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); +} + +/** ZSTD_ldm_insertEntry() : + * Insert the entry with corresponding hash into the hash table */ +static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, + size_t const hash, const ldmEntry_t entry, + ldmParams_t const ldmParams) +{ + BYTE* const pOffset = ldmState->bucketOffsets + hash; + unsigned const offset = *pOffset; + + *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry; + *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1)); + +} + +/** ZSTD_ldm_countBackwardsMatch() : + * Returns the number of bytes that match backwards before pIn and pMatch. + * + * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ +static size_t ZSTD_ldm_countBackwardsMatch( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase) +{ + size_t matchLength = 0; + while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { + pIn--; + pMatch--; + matchLength++; + } + return matchLength; +} + +/** ZSTD_ldm_countBackwardsMatch_2segments() : + * Returns the number of bytes that match backwards from pMatch, + * even with the backwards match spanning 2 different segments. + * + * On reaching `pMatchBase`, start counting from mEnd */ +static size_t ZSTD_ldm_countBackwardsMatch_2segments( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, + const BYTE* pExtDictStart, const BYTE* pExtDictEnd) +{ + size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { + /* If backwards match is entirely in the extDict or prefix, immediately return */ + return matchLength; + } + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); + matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); + DEBUGLOG(7, "final backwards match length = %zu", matchLength); + return matchLength; +} + +/** ZSTD_ldm_fillFastTables() : + * + * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. + * This is similar to ZSTD_loadDictionaryContent. + * + * The tables for the other strategies are filled within their + * block compressors. */ +static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, + void const* end) +{ + const BYTE* const iend = (const BYTE*)end; + + switch(ms->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); + break; + + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + default: + assert(0); /* not possible : not a valid strategy id */ + } + + return 0; +} + +void ZSTD_ldm_fillHashTable( + ldmState_t* ldmState, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params) +{ + U32 const minMatchLength = params->minMatchLength; + U32 const hBits = params->hashLog - params->bucketSizeLog; + BYTE const* const base = ldmState->window.base; + BYTE const* const istart = ip; + ldmRollingHashState_t hashState; + size_t* const splits = ldmState->splitIndices; + unsigned numSplits; + + DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); + + ZSTD_ldm_gear_init(&hashState, params); + while (ip < iend) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + if (ip + splits[n] >= istart + minMatchLength) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + ldmEntry_t entry; + + entry.offset = (U32)(split - base); + entry.checksum = (U32)(xxhash >> 32); + ZSTD_ldm_insertEntry(ldmState, hash, entry, *params); + } + } + + ip += hashed; + } +} + + +/** ZSTD_ldm_limitTableUpdate() : + * + * Sets cctx->nextToUpdate to a position corresponding closer to anchor + * if it is far way + * (after a long match, only update tables a limited amount). */ +static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) +{ + U32 const curr = (U32)(anchor - ms->window.base); + if (curr > ms->nextToUpdate + 1024) { + ms->nextToUpdate = + curr - MIN(512, curr - ms->nextToUpdate - 1024); + } +} + +static size_t ZSTD_ldm_generateSequences_internal( + ldmState_t* ldmState, rawSeqStore_t* rawSeqStore, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + /* LDM parameters */ + int const extDict = ZSTD_window_hasExtDict(ldmState->window); + U32 const minMatchLength = params->minMatchLength; + U32 const entsPerBucket = 1U << params->bucketSizeLog; + U32 const hBits = params->hashLog - params->bucketSizeLog; + /* Prefix and extDict parameters */ + U32 const dictLimit = ldmState->window.dictLimit; + U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; + BYTE const* const base = ldmState->window.base; + BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL; + BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL; + BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL; + BYTE const* const lowPrefixPtr = base + dictLimit; + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + BYTE const* const ilimit = iend - HASH_READ_SIZE; + /* Input positions */ + BYTE const* anchor = istart; + BYTE const* ip = istart; + /* Rolling hash state */ + ldmRollingHashState_t hashState; + /* Arrays for staged-processing */ + size_t* const splits = ldmState->splitIndices; + ldmMatchCandidate_t* const candidates = ldmState->matchCandidates; + unsigned numSplits; + + if (srcSize < minMatchLength) + return iend - anchor; + + /* Initialize the rolling hash state with the first minMatchLength bytes */ + ZSTD_ldm_gear_init(&hashState, params); + ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength); + ip += minMatchLength; + + while (ip < ilimit) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip, + splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + + candidates[n].split = split; + candidates[n].hash = hash; + candidates[n].checksum = (U32)(xxhash >> 32); + candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params); + PREFETCH_L1(candidates[n].bucket); + } + + for (n = 0; n < numSplits; n++) { + size_t forwardMatchLength = 0, backwardMatchLength = 0, + bestMatchLength = 0, mLength; + U32 offset; + BYTE const* const split = candidates[n].split; + U32 const checksum = candidates[n].checksum; + U32 const hash = candidates[n].hash; + ldmEntry_t* const bucket = candidates[n].bucket; + ldmEntry_t const* cur; + ldmEntry_t const* bestEntry = NULL; + ldmEntry_t newEntry; + + newEntry.offset = (U32)(split - base); + newEntry.checksum = checksum; + + /* If a split point would generate a sequence overlapping with + * the previous one, we merely register it in the hash table and + * move on */ + if (split < anchor) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + for (cur = bucket; cur < bucket + entsPerBucket; cur++) { + size_t curForwardMatchLength, curBackwardMatchLength, + curTotalMatchLength; + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + if (extDict) { + BYTE const* const curMatchBase = + cur->offset < dictLimit ? dictBase : base; + BYTE const* const pMatch = curMatchBase + cur->offset; + BYTE const* const matchEnd = + cur->offset < dictLimit ? dictEnd : iend; + BYTE const* const lowMatchPtr = + cur->offset < dictLimit ? dictStart : lowPrefixPtr; + curForwardMatchLength = + ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments( + split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd); + } else { /* !extDict */ + BYTE const* const pMatch = base + cur->offset; + curForwardMatchLength = ZSTD_count(split, pMatch, iend); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = + ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr); + } + curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; + + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } + + /* No match found -- insert an entry into the hash table + * and process the next candidate match */ + if (bestEntry == NULL) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + /* Match found */ + offset = (U32)(split - base) - bestEntry->offset; + mLength = forwardMatchLength + backwardMatchLength; + { + rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; + + /* Out of sequence storage */ + if (rawSeqStore->size == rawSeqStore->capacity) + return ERROR(dstSize_tooSmall); + seq->litLength = (U32)(split - backwardMatchLength - anchor); + seq->matchLength = (U32)mLength; + seq->offset = offset; + rawSeqStore->size++; + } + + /* Insert the current entry into the hash table --- it must be + * done after the previous block to avoid clobbering bestEntry */ + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + + anchor = split + forwardMatchLength; + + /* If we find a match that ends after the data that we've hashed + * then we have a repeating, overlapping, pattern. E.g. all zeros. + * If one repetition of the pattern matches our `stopMask` then all + * repetitions will. We don't need to insert them all into out table, + * only the first one. So skip over overlapping matches. + * This is a major speed boost (20x) for compressing a single byte + * repeated, when that byte ends up in the table. + */ + if (anchor > ip + hashed) { + ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength); + /* Continue the outer loop at anchor (ip + hashed == anchor). */ + ip = anchor - hashed; + break; + } + } + + ip += hashed; + } + + return iend - anchor; +} + +/*! ZSTD_ldm_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, + U32 const reducerValue) +{ + U32 u; + for (u = 0; u < size; u++) { + if (table[u].offset < reducerValue) table[u].offset = 0; + else table[u].offset -= reducerValue; + } +} + +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldmState, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + U32 const maxDist = 1U << params->windowLog; + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + size_t const kMaxChunkSize = 1 << 20; + size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0); + size_t chunk; + size_t leftoverSize = 0; + + assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize); + /* Check that ZSTD_window_update() has been called for this chunk prior + * to passing it to this function. + */ + assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); + /* The input could be very large (in zstdmt), so it must be broken up into + * chunks to enforce the maximum distance and handle overflow correction. + */ + assert(sequences->pos <= sequences->size); + assert(sequences->size <= sequences->capacity); + for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) { + BYTE const* const chunkStart = istart + chunk * kMaxChunkSize; + size_t const remaining = (size_t)(iend - chunkStart); + BYTE const *const chunkEnd = + (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize; + size_t const chunkSize = chunkEnd - chunkStart; + size_t newLeftoverSize; + size_t const prevSize = sequences->size; + + assert(chunkStart < iend); + /* 1. Perform overflow correction if necessary. */ + if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) { + U32 const ldmHSize = 1U << params->hashLog; + U32 const correction = ZSTD_window_correctOverflow( + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); + ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); + /* invalidate dictionaries on overflow correction */ + ldmState->loadedDictEnd = 0; + } + /* 2. We enforce the maximum offset allowed. + * + * kMaxChunkSize should be small enough that we don't lose too much of + * the window through early invalidation. + * TODO: * Test the chunk size. + * * Try invalidation after the sequence generation and test the + * offset against maxDist directly. + * + * NOTE: Because of dictionaries + sequence splitting we MUST make sure + * that any offset used is valid at the END of the sequence, since it may + * be split into two sequences. This condition holds when using + * ZSTD_window_enforceMaxDist(), but if we move to checking offsets + * against maxDist directly, we'll have to carefully handle that case. + */ + ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL); + /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ + newLeftoverSize = ZSTD_ldm_generateSequences_internal( + ldmState, sequences, params, chunkStart, chunkSize); + if (ZSTD_isError(newLeftoverSize)) + return newLeftoverSize; + /* 4. We add the leftover literals from previous iterations to the first + * newly generated sequence, or add the `newLeftoverSize` if none are + * generated. + */ + /* Prepend the leftover literals from the last call */ + if (prevSize < sequences->size) { + sequences->seq[prevSize].litLength += (U32)leftoverSize; + leftoverSize = newLeftoverSize; + } else { + assert(newLeftoverSize == chunkSize); + leftoverSize += chunkSize; + } + } + return 0; +} + +void +ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) +{ + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; + } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} + +/** + * If the sequence length is longer than remaining then the sequence is split + * between this block and the next. + * + * Returns the current sequence to handle, or if the rest of the block should + * be literals, it returns a sequence with offset == 0. + */ +static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, + U32 const remaining, U32 const minMatch) +{ + rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; + assert(sequence.offset > 0); + /* Likely: No partial sequence */ + if (remaining >= sequence.litLength + sequence.matchLength) { + rawSeqStore->pos++; + return sequence; + } + /* Cut the sequence short (offset == 0 ==> rest is literals). */ + if (remaining <= sequence.litLength) { + sequence.offset = 0; + } else if (remaining < sequence.litLength + sequence.matchLength) { + sequence.matchLength = remaining - sequence.litLength; + if (sequence.matchLength < minMatch) { + sequence.offset = 0; + } + } + /* Skip past `remaining` bytes for the future sequences. */ + ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); + return sequence; +} + +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_paramSwitch_e useRowMatchFinder, + void const* src, size_t srcSize) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + unsigned const minMatch = cParams->minMatch; + ZSTD_blockCompressor const blockCompressor = + ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms)); + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + /* Input positions */ + BYTE const* ip = istart; + + DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ + if (cParams->strategy >= ZSTD_btopt) { + size_t lastLLSize; + ms->ldmSeqStore = rawSeqStore; + lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); + return lastLLSize; + } + + assert(rawSeqStore->pos <= rawSeqStore->size); + assert(rawSeqStore->size <= rawSeqStore->capacity); + /* Loop through each sequence and apply the block compressor to the literals */ + while (rawSeqStore->pos < rawSeqStore->size && ip < iend) { + /* maybeSplitSequence updates rawSeqStore->pos */ + rawSeq const sequence = maybeSplitSequence(rawSeqStore, + (U32)(iend - ip), minMatch); + int i; + /* End signal */ + if (sequence.offset == 0) + break; + + assert(ip + sequence.litLength + sequence.matchLength <= iend); + + /* Fill tables for block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Run the block compressor */ + DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); + { + size_t const newLitLength = + blockCompressor(ms, seqStore, rep, ip, sequence.litLength); + ip += sequence.litLength; + /* Update the repcodes */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + rep[i] = rep[i-1]; + rep[0] = sequence.offset; + /* Store the sequence */ + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, + OFFSET_TO_OFFBASE(sequence.offset), + sequence.matchLength); + ip += sequence.matchLength; + } + } + /* Fill the tables for the block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Compress the last literals */ + return blockCompressor(ms, seqStore, rep, ip, iend - ip); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h new file mode 100644 index 000000000..f147021d2 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_H +#define ZSTD_LDM_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" /* ldmParams_t, U32 */ +#include "../zstd.h" /* ZSTD_CCtx, size_t */ + +/*-************************************* +* Long distance matching +***************************************/ + +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT + +void ZSTD_ldm_fillHashTable( + ldmState_t* state, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params); + +/** + * ZSTD_ldm_generateSequences(): + * + * Generates the sequences using the long distance match finder. + * Generates long range matching sequences in `sequences`, which parse a prefix + * of the source. `sequences` must be large enough to store every sequence, + * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. + * @returns 0 or an error code. + * + * NOTE: The user must have called ZSTD_window_update() for all of the input + * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. + * NOTE: This function returns an error if it runs out of space to store + * sequences. + */ +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldms, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize); + +/** + * ZSTD_ldm_blockCompress(): + * + * Compresses a block using the predefined sequences, along with a secondary + * block compressor. The literals section of every sequence is passed to the + * secondary block compressor, and those sequences are interspersed with the + * predefined sequences. Returns the length of the last literals. + * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. + * `rawSeqStore.seq` may also be updated to split the last sequence between two + * blocks. + * @return The length of the last literals. + * + * NOTE: The source must be at most the maximum block size, but the predefined + * sequences can be any size, and may be longer than the block. In the case that + * they are longer than the block, the last sequences may need to be split into + * two. We handle that case correctly, and update `rawSeqStore` appropriately. + * NOTE: This function does not return any errors. + */ +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_paramSwitch_e useRowMatchFinder, + void const* src, size_t srcSize); + +/** + * ZSTD_ldm_skipSequences(): + * + * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. + * Avoids emitting matches less than `minMatch` bytes. + * Must be called for data that is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, + U32 const minMatch); + +/* ZSTD_ldm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. + * Not to be used in conjunction with ZSTD_ldm_skipSequences(). + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes); + +/** ZSTD_ldm_getTableSize() : + * Estimate the space needed for long distance matching tables or 0 if LDM is + * disabled. + */ +size_t ZSTD_ldm_getTableSize(ldmParams_t params); + +/** ZSTD_ldm_getSeqSpace() : + * Return an upper bound on the number of sequences that can be produced by + * the long distance matcher, or 0 if LDM is disabled. + */ +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); + +/** ZSTD_ldm_adjustParameters() : + * If the params->hashRateLog is not set, set it to its default value based on + * windowLog and params->hashLog. + * + * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to + * params->hashLog if it is not). + * + * Ensures that the minMatchLength >= targetLength during optimal parsing. + */ +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm_geartab.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm_geartab.h new file mode 100644 index 000000000..ef34bc5c9 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm_geartab.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_GEARTAB_H +#define ZSTD_LDM_GEARTAB_H + +#include "../common/compiler.h" /* UNUSED_ATTR */ +#include "../common/mem.h" /* U64 */ + +static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = { + 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc, + 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05, + 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e, + 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889, + 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e, + 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140, + 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e, + 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f, + 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391, + 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210, + 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be, + 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a, + 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b, + 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4, + 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb, + 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312, + 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01, + 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc, + 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967, + 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553, + 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f, + 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2, + 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d, + 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a, + 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74, + 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3, + 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1, + 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b, + 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568, + 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a, + 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1, + 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9, + 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463, + 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba, + 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9, + 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61, + 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec, + 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6, + 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab, + 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5, + 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59, + 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7, + 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc, + 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb, + 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be, + 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312, + 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1, + 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc, + 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d, + 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445, + 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c, + 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5, + 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5, + 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28, + 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a, + 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9, + 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15, + 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef, + 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2, + 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375, + 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3, + 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595, + 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389, + 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4, + 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756, + 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc, + 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45, + 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea, + 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f, + 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc, + 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c, + 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a, + 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17, + 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3, + 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4, + 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91, + 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40, + 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741, + 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f, + 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4, + 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad, + 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047, + 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2, + 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e, + 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b, + 0x2b4da14f2613d8f4 +}; + +#endif /* ZSTD_LDM_GEARTAB_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.c new file mode 100644 index 000000000..f02a76094 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.c @@ -0,0 +1,1472 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "hist.h" +#include "zstd_opt.h" + + +#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ +#define ZSTD_MAX_PRICE (1<<30) + +#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ + + +/*-************************************* +* Price functions for optimal parser +***************************************/ + +#if 0 /* approximation at bit level (for tests) */ +# define BITCOST_ACCURACY 0 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy (for tests) */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat)) +#else /* opt==approx, ultra==accurate */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +#endif + +/* ZSTD_bitWeight() : + * provide estimated "cost" of a stat in full bits only */ +MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +{ + return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); +} + +/* ZSTD_fracWeight() : + * provide fractional-bit "cost" of a stat, + * using linear interpolation approximation */ +MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) +{ + U32 const stat = rawStat + 1; + U32 const hb = ZSTD_highbit32(stat); + U32 const BWeight = hb * BITCOST_MULTIPLIER; + /* Fweight was meant for "Fractional weight" + * but it's effectively a value between 1 and 2 + * using fixed point arithmetic */ + U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + BITCOST_ACCURACY < 31); + return weight; +} + +#if (DEBUGLEVEL>=2) +/* debugging function, + * @return price in bytes as fractional value + * for debug messages only */ +MEM_STATIC double ZSTD_fCost(int price) +{ + return (double)price / (BITCOST_MULTIPLIER*8); +} +#endif + +static int ZSTD_compressedLiterals(optState_t const* const optPtr) +{ + return optPtr->literalCompressionMode != ZSTD_ps_disable; +} + +static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) +{ + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); + optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); + optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); +} + + +static U32 sum_u32(const unsigned table[], size_t nbElts) +{ + size_t n; + U32 total = 0; + for (n=0; n0); + unsigned const newStat = base + (table[s] >> shift); + sum += newStat; + table[s] = newStat; + } + return sum; +} + +/* ZSTD_scaleStats() : + * reduce all elt frequencies in table if sum too large + * return the resulting sum of elements */ +static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) +{ + U32 const prevsum = sum_u32(table, lastEltIndex+1); + U32 const factor = prevsum >> logTarget; + DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); + assert(logTarget < 30); + if (factor <= 1) return prevsum; + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed); +} + +/* ZSTD_rescaleFreqs() : + * if first block (detected by optPtr->litLengthSum == 0) : init statistics + * take hints from dictionary if there is one + * and init from zero if there is none, + * using src for literals stats, and baseline stats for sequence symbols + * otherwise downscale existing stats, to be used as seed for next block. + */ +static void +ZSTD_rescaleFreqs(optState_t* const optPtr, + const BYTE* const src, size_t const srcSize, + int const optLevel) +{ + int const compressedLiterals = ZSTD_compressedLiterals(optPtr); + DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); + optPtr->priceType = zop_dynamic; + + if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */ + + /* heuristic: use pre-defined stats for too small inputs */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { + DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD); + optPtr->priceType = zop_predef; + } + + assert(optPtr->symbolCosts != NULL); + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { + + /* huffman stats covering the full value set : table presumed generated by dictionary */ + optPtr->priceType = zop_dynamic; + + if (compressedLiterals) { + /* generate literals statistics from huffman table */ + unsigned lit; + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; + for (lit=0; lit<=MaxLit; lit++) { + U32 const scaleLog = 11; /* scale to 2K */ + U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit); + assert(bitCost <= scaleLog); + optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litSum += optPtr->litFreq[lit]; + } } + + { unsigned ll; + FSE_CState_t llstate; + FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); + optPtr->litLengthSum = 0; + for (ll=0; ll<=MaxLL; ll++) { + U32 const scaleLog = 10; /* scale to 1K */ + U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); + assert(bitCost < scaleLog); + optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litLengthSum += optPtr->litLengthFreq[ll]; + } } + + { unsigned ml; + FSE_CState_t mlstate; + FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); + optPtr->matchLengthSum = 0; + for (ml=0; ml<=MaxML; ml++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); + assert(bitCost < scaleLog); + optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; + } } + + { unsigned of; + FSE_CState_t ofstate; + FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); + optPtr->offCodeSum = 0; + for (of=0; of<=MaxOff; of++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); + assert(bitCost < scaleLog); + optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->offCodeSum += optPtr->offCodeFreq[of]; + } } + + } else { /* first block, no dictionary */ + + assert(optPtr->litFreq != NULL); + if (compressedLiterals) { + /* base initial cost of literals on direct frequency within src */ + unsigned lit = MaxLit; + HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible); + } + + { unsigned const baseLLfreqs[MaxLL+1] = { + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); + optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); + } + + { unsigned ml; + for (ml=0; ml<=MaxML; ml++) + optPtr->matchLengthFreq[ml] = 1; + } + optPtr->matchLengthSum = MaxML+1; + + { unsigned const baseOFCfreqs[MaxOff+1] = { + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); + optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); + } + + } + + } else { /* new block : scale down accumulated statistics */ + + if (compressedLiterals) + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); + } + + ZSTD_setBasePrices(optPtr, optLevel); +} + +/* ZSTD_rawLiteralsCost() : + * price of literals (only) in specified segment (which length can be 0). + * does not include price of literalLength symbol */ +static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, + const optState_t* const optPtr, + int optLevel) +{ + if (litLength == 0) return 0; + + if (!ZSTD_compressedLiterals(optPtr)) + return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ + + if (optPtr->priceType == zop_predef) + return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ + + /* dynamic statistics */ + { U32 price = optPtr->litSumBasePrice * litLength; + U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER; + U32 u; + assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER); + for (u=0; u < litLength; u++) { + U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel); + if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax; + price -= litPrice; + } + return price; + } +} + +/* ZSTD_litLengthPrice() : + * cost of literalLength symbol */ +static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + assert(litLength <= ZSTD_BLOCKSIZE_MAX); + if (optPtr->priceType == zop_predef) + return WEIGHT(litLength, optLevel); + + /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX + * because it isn't representable in the zstd format. + * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. + * In such a case, the block would be all literals. + */ + if (litLength == ZSTD_BLOCKSIZE_MAX) + return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel); + + /* dynamic statistics */ + { U32 const llCode = ZSTD_LLcode(litLength); + return (LL_bits[llCode] * BITCOST_MULTIPLIER) + + optPtr->litLengthSumBasePrice + - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + } +} + +/* ZSTD_getMatchPrice() : + * Provides the cost of the match part (offset + matchLength) of a sequence. + * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. + * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq() + * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_getMatchPrice(U32 const offBase, + U32 const matchLength, + const optState_t* const optPtr, + int const optLevel) +{ + U32 price; + U32 const offCode = ZSTD_highbit32(offBase); + U32 const mlBase = matchLength - MINMATCH; + assert(matchLength >= MINMATCH); + + if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */ + return WEIGHT(mlBase, optLevel) + + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */ + + /* dynamic statistics */ + price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); + if ((optLevel<2) /*static*/ && offCode >= 20) + price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ + + /* match Length */ + { U32 const mlCode = ZSTD_MLcode(mlBase); + price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); + } + + price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ + + DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); + return price; +} + +/* ZSTD_updateStats() : + * assumption : literals + litLength <= iend */ +static void ZSTD_updateStats(optState_t* const optPtr, + U32 litLength, const BYTE* literals, + U32 offBase, U32 matchLength) +{ + /* literals */ + if (ZSTD_compressedLiterals(optPtr)) { + U32 u; + for (u=0; u < litLength; u++) + optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; + } + + /* literal Length */ + { U32 const llCode = ZSTD_LLcode(litLength); + optPtr->litLengthFreq[llCode]++; + optPtr->litLengthSum++; + } + + /* offset code : follows storeSeq() numeric representation */ + { U32 const offCode = ZSTD_highbit32(offBase); + assert(offCode <= MaxOff); + optPtr->offCodeFreq[offCode]++; + optPtr->offCodeSum++; + } + + /* match Length */ + { U32 const mlBase = matchLength - MINMATCH; + U32 const mlCode = ZSTD_MLcode(mlBase); + optPtr->matchLengthFreq[mlCode]++; + optPtr->matchLengthSum++; + } +} + + +/* ZSTD_readMINMATCH() : + * function safe only for comparisons + * assumption : memPtr must be at least 4 bytes before end of buffer */ +MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) +{ + U32* const hashTable3 = ms->hashTable3; + U32 const hashLog3 = ms->hashLog3; + const BYTE* const base = ms->window.base; + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); + size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); + assert(hashLog3 > 0); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + *nextToUpdate3 = target; + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. + * @param ip assumed <= iend-8 . + * @param target The target of ZSTD_updateTree_internal() - we are filling to this position + * @return : nb of positions added */ +static U32 ZSTD_insertBt1( + const ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + U32 const target, + U32 const mls, const int extDict) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 curr = (U32)(ip-base); + const U32 btLow = btMask >= curr ? 0 : curr - btMask; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + /* windowLow is based on target because + * we only need positions that will be in the window at the end of the tree update. + */ + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, target, cParams->windowLog); + U32 matchEndIdx = curr+8+1; + size_t bestLength = 8; + U32 nbCompares = 1U << cParams->searchLog; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ + + DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); + + assert(curr <= target); + assert(ip <= iend-8); /* required for h calculation */ + hashTable[h] = curr; /* Update Hash Table */ + + assert(windowLow > 0); + for (; nbCompares && (matchIndex >= windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + + if (!extDict || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > curr + 8); + return MAX(positions, matchEndIdx - (curr + 8)); + } +} + +FORCE_INLINE_TEMPLATE +void ZSTD_updateTree_internal( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + idx, target, dictMode); + + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, target, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; + } + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); + ms->nextToUpdate = target; +} + +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { + ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); +} + +FORCE_INLINE_TEMPLATE U32 +ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip, const BYTE* const iLimit, + const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + const U32 lengthToBeat, + const U32 mls /* template */) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const hashLog = cParams->hashLog; + U32 const minMatch = (mls==3) ? 3 : 4; + U32* const hashTable = ms->hashTable; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = ms->window.dictBase; + U32 const dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + U32 const matchLow = windowLow ? windowLow : 1; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + U32 nbCompares = 1U << cParams->searchLog; + + const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; + const ZSTD_compressionParameters* const dmsCParams = + dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; + U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; + U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; + U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; + U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; + + size_t bestLength = lengthToBeat-1; + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); + + /* check repCode */ + assert(ll0 <= 1); /* necessarily 1 or 0 */ + { U32 const lastR = ZSTD_REP_NUM + ll0; + U32 repCode; + for (repCode = ll0; repCode < lastR; repCode++) { + U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + U32 const repIndex = curr - repOffset; + U32 repLen = 0; + assert(curr >= dictLimit); + if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ + /* We must validate the repcode offset because when we're using a dictionary the + * valid offset range shrinks when the dictionary goes out of bounds. + */ + if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { + repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; + } + } else { /* repIndex < dictLimit || repIndex >= curr */ + const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? + dmsBase + repIndex - dmsIndexDelta : + dictBase + repIndex; + assert(curr >= windowLow); + if ( dictMode == ZSTD_extDict + && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ + & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } + if (dictMode == ZSTD_dictMatchState + && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ + & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; + } } + /* save longer solution */ + if (repLen > bestLength) { + DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", + repCode, ll0, repOffset, repLen); + bestLength = repLen; + matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */ + matches[mnum].len = (U32)repLen; + mnum++; + if ( (repLen > sufficient_len) + | (ip+repLen == iLimit) ) { /* best possible */ + return mnum; + } } } } + + /* HC3 match finder */ + if ((mls == 3) /*static*/ && (bestLength < mls)) { + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); + if ((matchIndex3 >= matchLow) + & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { + size_t mlen; + if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { + const BYTE* const match = base + matchIndex3; + mlen = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex3; + mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); + } + + /* save best solution */ + if (mlen >= mls /* == 3 > bestLength */) { + DEBUGLOG(8, "found small match with hlog3, of length %u", + (U32)mlen); + bestLength = mlen; + assert(curr > matchIndex3); + assert(mnum==0); /* no prior solution */ + matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3); + matches[0].len = (U32)mlen; + mnum = 1; + if ( (mlen > sufficient_len) | + (ip+mlen == iLimit) ) { /* best possible length */ + ms->nextToUpdate = curr+1; /* skip insertion */ + return 1; + } } } + /* no dictMatchState lookup: dicts don't have a populated HC3 table */ + } /* if (mls == 3) */ + + hashTable[h] = curr; /* Update Hash Table */ + + for (; nbCompares && (matchIndex >= matchLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + const BYTE* match; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(curr > matchIndex); + + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ + match = base + matchIndex; + if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); + } else { + match = dictBase + matchIndex; + assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* prepare for match[matchLength] read */ + } + + if (matchLength > bestLength) { + DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); + assert(matchEndIdx > matchIndex); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ + break; /* drop, to preserve bt consistency (miss a little bit of compression) */ + } } + + if (match[matchLength] < ip[matchLength]) { + /* match smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ + } else { + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); + U32 dictMatchIndex = dms->hashTable[dmsH]; + const U32* const dmsBt = dms->chainTable; + commonLengthSmaller = commonLengthLarger = 0; + for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) { + const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dmsBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); + if (dictMatchIndex+matchLength >= dmsHighLimit) + match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + matchIndex = dictMatchIndex + dmsIndexDelta; + DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } } + + if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ + if (match[matchLength] < ip[matchLength]) { + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } } } /* if (dictMode == ZSTD_dictMatchState) */ + + assert(matchEndIdx > curr+8); + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + return mnum; +} + +typedef U32 (*ZSTD_getAllMatchesFn)( + ZSTD_match_t*, + ZSTD_matchState_t*, + U32*, + const BYTE*, + const BYTE*, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat); + +FORCE_INLINE_TEMPLATE U32 ZSTD_btGetAllMatches_internal( + ZSTD_match_t* matches, + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* ip, + const BYTE* const iHighLimit, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat, + const ZSTD_dictMode_e dictMode, + const U32 mls) +{ + assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls); + DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls); + if (ip < ms->window.base + ms->nextToUpdate) + return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode); + return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls); +} + +#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls + +#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \ + static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \ + ZSTD_match_t* matches, \ + ZSTD_matchState_t* ms, \ + U32* nextToUpdate3, \ + const BYTE* ip, \ + const BYTE* const iHighLimit, \ + const U32 rep[ZSTD_REP_NUM], \ + U32 const ll0, \ + U32 const lengthToBeat) \ + { \ + return ZSTD_btGetAllMatches_internal( \ + matches, ms, nextToUpdate3, ip, iHighLimit, \ + rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \ + } + +#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6) + +GEN_ZSTD_BT_GET_ALL_MATCHES(noDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(extDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState) + +#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \ + { \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \ + } + +static ZSTD_getAllMatchesFn +ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const dictMode) +{ + ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = { + ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState) + }; + U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6); + assert((U32)dictMode < 3); + assert(mls - 3 < 4); + return getAllMatchesFns[(int)dictMode][mls - 3]; +} + +/************************* +* LDM helper functions * +*************************/ + +/* Struct containing info needed to make decision about ldm inclusion */ +typedef struct { + rawSeqStore_t seqStore; /* External match candidates store for this block */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ +} ZSTD_optLdm_t; + +/* ZSTD_optLdm_skipRawSeqStoreBytes(): + * Moves forward in @rawSeqStore by @nbBytes, + * which will update the fields 'pos' and 'posInSequence'. + */ +static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) +{ + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): + * Calculates the beginning and end of the next match in the current block. + * Updates 'pos' and 'posInSequence' of the ldmSeqStore. + */ +static void +ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) +{ + rawSeq currSeq; + U32 currBlockEndPos; + U32 literalsBytesRemaining; + U32 matchBytesRemaining; + + /* Setting match end position to MAX to ensure we never use an LDM during this block */ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + return; + } + /* Calculate appropriate bytes left in matchLength and litLength + * after adjusting based on ldmSeqStore->posInSequence */ + currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; + assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); + currBlockEndPos = currPosInBlock + blockBytesRemaining; + literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore.posInSequence : + 0; + matchBytesRemaining = (literalsBytesRemaining == 0) ? + currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : + currSeq.matchLength; + + /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ + if (literalsBytesRemaining >= blockBytesRemaining) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); + return; + } + + /* Matches may be < MINMATCH by this process. In that case, we will reject them + when we are deciding whether or not to add the ldm */ + optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; + optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; + optLdm->offset = currSeq.offset; + + if (optLdm->endPosInBlock > currBlockEndPos) { + /* Match ends after the block ends, we can't use the whole match */ + optLdm->endPosInBlock = currBlockEndPos; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); + } else { + /* Consume nb of bytes equal to size of sequence left */ + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); + } +} + +/* ZSTD_optLdm_maybeAddMatch(): + * Adds a match if it's long enough, + * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock', + * into 'matches'. Maintains the correct ordering of 'matches'. + */ +static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, + const ZSTD_optLdm_t* optLdm, U32 currPosInBlock) +{ + U32 const posDiff = currPosInBlock - optLdm->startPosInBlock; + /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */ + U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; + + /* Ensure that current block position is not outside of the match */ + if (currPosInBlock < optLdm->startPosInBlock + || currPosInBlock >= optLdm->endPosInBlock + || candidateMatchLength < MINMATCH) { + return; + } + + if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { + U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset); + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u", + candidateOffBase, candidateMatchLength, currPosInBlock); + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffBase; + (*nbMatches)++; + } +} + +/* ZSTD_optLdm_processMatchCandidate(): + * Wrapper function to update ldm seq store and call ldm functions as necessary. + */ +static void +ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, + ZSTD_match_t* matches, U32* nbMatches, + U32 currPosInBlock, U32 remainingBytes) +{ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + return; + } + + if (currPosInBlock >= optLdm->endPosInBlock) { + if (currPosInBlock > optLdm->endPosInBlock) { + /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily + * at the end of a match from the ldm seq store, and will often be some bytes + * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" + */ + U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); + } + ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); + } + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); +} + + +/*-******************************* +* Optimal parser +*********************************/ + +static U32 ZSTD_totalLen(ZSTD_optimal_t sol) +{ + return sol.litlen + sol.mlen; +} + +#if 0 /* debug */ + +static void +listStats(const U32* table, int lastEltID) +{ + int const nbElts = lastEltID + 1; + int enb; + for (enb=0; enb < nbElts; enb++) { + (void)table; + /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ + RAWLOG(2, "%4i,", table[enb]); + } + RAWLOG(2, " \n"); +} + +#endif + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const int optLevel, + const ZSTD_dictMode_e dictMode) +{ + optState_t* const optStatePtr = &ms->opt; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + + ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode); + + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; + + ZSTD_optimal_t* const opt = optStatePtr->priceTable; + ZSTD_match_t* const matches = optStatePtr->matchTable; + ZSTD_optimal_t lastSequence; + ZSTD_optLdm_t optLdm; + + ZSTD_memset(&lastSequence, 0, sizeof(ZSTD_optimal_t)); + + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; + optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", + (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); + assert(optLevel <= 2); + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); + ip += (ip==prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, last_pos = 0; + + /* find first match */ + { U32 const litlen = (U32)(ip - anchor); + U32 const ll0 = !litlen; + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch); + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(ip-istart), (U32)(iend - ip)); + if (!nbMatches) { ip++; continue; } + + /* initialize opt[0] */ + { U32 i ; for (i=0; i immediate encoding */ + { U32 const maxML = matches[nbMatches-1].len; + U32 const maxOffBase = matches[nbMatches-1].off; + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series", + nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart)); + + if (maxML > sufficient_len) { + lastSequence.litlen = litlen; + lastSequence.mlen = maxML; + lastSequence.off = maxOffBase; + DEBUGLOG(6, "large match (%u>%u), immediate encoding", + maxML, sufficient_len); + cur = 0; + last_pos = ZSTD_totalLen(lastSequence); + goto _shortestPath; + } } + + /* set prices for first matches starting position == 0 */ + assert(opt[0].price >= 0); + { U32 const literalsPrice = (U32)opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 pos; + U32 matchNb; + for (pos = 1; pos < minMatch; pos++) { + opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ + } + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offBase = matches[matchNb].off; + U32 const end = matches[matchNb].len; + for ( ; pos <= end ; pos++ ) { + U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel); + U32 const sequencePrice = literalsPrice + matchPrice; + DEBUGLOG(7, "rPos:%u => set initial price : %.2f", + pos, ZSTD_fCost((int)sequencePrice)); + opt[pos].mlen = pos; + opt[pos].off = offBase; + opt[pos].litlen = litlen; + opt[pos].price = (int)sequencePrice; + } } + last_pos = pos-1; + } + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + const BYTE* const inr = ip + cur; + assert(cur < ZSTD_OPT_NUM); + DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur) + + /* Fix current position with one literal if cheaper */ + { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; + int const price = opt[cur-1].price + + (int)ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) + + (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) + - (int)ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); + assert(price < 1000000000); /* overflow check */ + if (price <= opt[cur].price) { + DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, + opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); + opt[cur].mlen = 0; + opt[cur].off = 0; + opt[cur].litlen = litlen; + opt[cur].price = price; + } else { + DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), + opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]); + } + } + + /* Set the repcodes of the current position. We must do it here + * because we rely on the repcodes of the 2nd to last sequence being + * correct to set the next chunks repcodes during the backward + * traversal. + */ + ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); + assert(cur >= opt[cur].mlen); + if (opt[cur].mlen != 0) { + U32 const prev = cur - opt[cur].mlen; + repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0); + ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); + } else { + ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); + } + + /* last match must start at a minimum distance of 8 from oend */ + if (inr > ilimit) continue; + + if (cur == last_pos) break; + + if ( (optLevel==0) /*static_test*/ + && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { + DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1); + continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ + } + + assert(opt[cur].price >= 0); + { U32 const ll0 = (opt[cur].mlen != 0); + U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; + U32 const previousPrice = (U32)opt[cur].price; + U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch); + U32 matchNb; + + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(inr-istart), (U32)(iend-inr)); + + if (!nbMatches) { + DEBUGLOG(7, "rPos:%u : no match found", cur); + continue; + } + + { U32 const maxML = matches[nbMatches-1].len; + DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u", + inr-istart, cur, nbMatches, maxML); + + if ( (maxML > sufficient_len) + || (cur + maxML >= ZSTD_OPT_NUM) ) { + lastSequence.mlen = maxML; + lastSequence.off = matches[nbMatches-1].off; + lastSequence.litlen = litlen; + cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */ + last_pos = cur + ZSTD_totalLen(lastSequence); + if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */ + goto _shortestPath; + } } + + /* set prices using matches found at position == cur */ + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + U32 const lastML = matches[matchNb].len; + U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; + U32 mlen; + + DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u", + matchNb, matches[matchNb].off, lastML, litlen); + + for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ + U32 const pos = cur + mlen; + int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + + if ((pos > last_pos) || (price < opt[pos].price)) { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */ + opt[pos].mlen = mlen; + opt[pos].off = offset; + opt[pos].litlen = litlen; + opt[pos].price = price; + } else { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ + } + } } } + } /* for (cur = 1; cur <= last_pos; cur++) */ + + lastSequence = opt[last_pos]; + cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */ + assert(cur < ZSTD_OPT_NUM); /* control overflow*/ + +_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ + assert(opt[0].mlen == 0); + + /* Set the next chunk's repcodes based on the repcodes of the beginning + * of the last match, and the last sequence. This avoids us having to + * update them while traversing the sequences. + */ + if (lastSequence.mlen != 0) { + repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0); + ZSTD_memcpy(rep, &reps, sizeof(reps)); + } else { + ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); + } + + { U32 const storeEnd = cur + 1; + U32 storeStart = storeEnd; + U32 seqPos = cur; + + DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", + last_pos, cur); (void)last_pos; + assert(storeEnd < ZSTD_OPT_NUM); + DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off); + opt[storeEnd] = lastSequence; + while (seqPos > 0) { + U32 const backDist = ZSTD_totalLen(opt[seqPos]); + storeStart--; + DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off); + opt[storeStart] = opt[seqPos]; + seqPos = (seqPos > backDist) ? seqPos - backDist : 0; + } + + /* save sequences */ + DEBUGLOG(6, "sending selected sequences into seqStore") + { U32 storePos; + for (storePos=storeStart; storePos <= storeEnd; storePos++) { + U32 const llen = opt[storePos].litlen; + U32 const mlen = opt[storePos].mlen; + U32 const offBase = opt[storePos].off; + U32 const advance = llen + mlen; + DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", + anchor - istart, (unsigned)llen, (unsigned)mlen); + + if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ + assert(storePos == storeEnd); /* must be last sequence */ + ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ + continue; /* will finish */ + } + + assert(anchor + llen <= iend); + ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen); + anchor += advance; + ip = anchor; + } } + ZSTD_setBasePrices(optStatePtr, optLevel); + } + } /* while (ip < ilimit) */ + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +static size_t ZSTD_compressBlock_opt0( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode); +} + +static size_t ZSTD_compressBlock_opt2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode); +} + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btopt"); + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + + + + +/* ZSTD_initStats_ultra(): + * make a first compression pass, just to seed stats with more accurate starting values. + * only works on first block, with no dictionary and no ldm. + * this function cannot error out, its narrow contract must be respected. + */ +static void +ZSTD_initStats_ultra(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); + + DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); + assert(ms->opt.litLengthSum == 0); /* first block */ + assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ + assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ + assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ + + ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/ + + /* invalidate first scan from history, only keep entropy stats */ + ZSTD_resetSeqStore(seqStore); + ms->window.base -= srcSize; + ms->window.dictLimit += (U32)srcSize; + ms->window.lowLimit = ms->window.dictLimit; + ms->nextToUpdate = ms->window.dictLimit; + +} + +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 const curr = (U32)((const BYTE*)src - ms->window.base); + DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + + /* 2-passes strategy: + * this strategy makes a first pass over first block to collect statistics + * in order to seed next round's statistics with it. + * After 1st pass, function forgets history, and starts a new block. + * Consequently, this can only work if no data has been previously loaded in tables, + * aka, no dictionary, no prefix, no ldm preprocessing. + * The compression ratio gain is generally small (~0.5% on first block), + ** the cost is 2x cpu time on first block. */ + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + if ( (ms->opt.litLengthSum==0) /* first block */ + && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ + && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */ + ) { + ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); + } + + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} + +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} + +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.h new file mode 100644 index 000000000..342e5a311 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_OPT_H +#define ZSTD_OPT_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +/* used in ZSTD_loadDictionaryContent() */ +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend); + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + /* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_OPT_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.c b/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.c new file mode 100644 index 000000000..678607556 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.c @@ -0,0 +1,1867 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +/* ====== Constants ====== */ +#define ZSTDMT_OVERLAPLOG_DEFAULT 0 + + +/* ====== Dependencies ====== */ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */ +#include "../common/mem.h" /* MEM_STATIC */ +#include "../common/pool.h" /* threadpool */ +#include "../common/threading.h" /* mutex */ +#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ +#include "zstd_ldm.h" +#include "zstdmt_compress.h" + +/* Guards code to support resizing the SeqPool. + * We will want to resize the SeqPool to save memory in the future. + * Until then, comment the code out since it is unused. + */ +#define ZSTD_RESIZE_SEQPOOL 0 + +/* ====== Debug ====== */ +#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2) \ + && !defined(_MSC_VER) \ + && !defined(__MINGW32__) + +# include +# include +# include + +# define DEBUG_PRINTHEX(l,p,n) { \ + unsigned debug_u; \ + for (debug_u=0; debug_u<(n); debug_u++) \ + RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ + RAWLOG(l, " \n"); \ +} + +static unsigned long long GetCurrentClockTimeMicroseconds(void) +{ + static clock_t _ticksPerSecond = 0; + if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); + + { struct tms junk; clock_t newTicks = (clock_t) times(&junk); + return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); +} } + +#define MUTEX_WAIT_TIME_DLEVEL 6 +#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \ + if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + ZSTD_pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ + elapsedTime, #mutex); \ + } } \ + } else { \ + ZSTD_pthread_mutex_lock(mutex); \ + } \ +} + +#else + +# define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) +# define DEBUG_PRINTHEX(l,p,n) {} + +#endif + + +/* ===== Buffer Pool ===== */ +/* a single Buffer Pool can be invoked from multiple threads in parallel */ + +typedef struct buffer_s { + void* start; + size_t capacity; +} buffer_t; + +static const buffer_t g_nullBuffer = { NULL, 0 }; + +typedef struct ZSTDMT_bufferPool_s { + ZSTD_pthread_mutex_t poolMutex; + size_t bufferSize; + unsigned totalBuffers; + unsigned nbBuffers; + ZSTD_customMem cMem; + buffer_t bTable[1]; /* variable size */ +} ZSTDMT_bufferPool; + +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_customMem cMem) +{ + ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_customCalloc( + sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem); + if (bufPool==NULL) return NULL; + if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { + ZSTD_customFree(bufPool, cMem); + return NULL; + } + bufPool->bufferSize = 64 KB; + bufPool->totalBuffers = maxNbBuffers; + bufPool->nbBuffers = 0; + bufPool->cMem = cMem; + return bufPool; +} + +static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) +{ + unsigned u; + DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); + if (!bufPool) return; /* compatibility with free on NULL */ + for (u=0; utotalBuffers; u++) { + DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start); + ZSTD_customFree(bufPool->bTable[u].start, bufPool->cMem); + } + ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); + ZSTD_customFree(bufPool, bufPool->cMem); +} + +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool) + + (bufPool->totalBuffers - 1) * sizeof(buffer_t); + unsigned u; + size_t totalBufferSize = 0; + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + for (u=0; utotalBuffers; u++) + totalBufferSize += bufPool->bTable[u].capacity; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + + return poolSize + totalBufferSize; +} + +/* ZSTDMT_setBufferSize() : + * all future buffers provided by this buffer pool will have _at least_ this size + * note : it's better for all buffers to have same size, + * as they become freely interchangeable, reducing malloc/free usages and memory fragmentation */ +static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize) +{ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); + bufPool->bufferSize = bSize; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); +} + + +static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, unsigned maxNbBuffers) +{ + if (srcBufPool==NULL) return NULL; + if (srcBufPool->totalBuffers >= maxNbBuffers) /* good enough */ + return srcBufPool; + /* need a larger buffer pool */ + { ZSTD_customMem const cMem = srcBufPool->cMem; + size_t const bSize = srcBufPool->bufferSize; /* forward parameters */ + ZSTDMT_bufferPool* newBufPool; + ZSTDMT_freeBufferPool(srcBufPool); + newBufPool = ZSTDMT_createBufferPool(maxNbBuffers, cMem); + if (newBufPool==NULL) return newBufPool; + ZSTDMT_setBufferSize(newBufPool, bSize); + return newBufPool; + } +} + +/** ZSTDMT_getBuffer() : + * assumption : bufPool must be valid + * @return : a buffer, with start pointer and size + * note: allocation may fail, in this case, start==NULL and size==0 */ +static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) +{ + size_t const bSize = bufPool->bufferSize; + DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers) { /* try to use an existing buffer */ + buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)]; + size_t const availBufferSize = buf.capacity; + bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer; + if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) { + /* large enough, but not too much */ + DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u", + bufPool->nbBuffers, (U32)buf.capacity); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return buf; + } + /* size conditions not respected : scratch this buffer, create new one */ + DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing"); + ZSTD_customFree(buf.start, bufPool->cMem); + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* create new buffer */ + DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); + { buffer_t buffer; + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer.start = start; /* note : start can be NULL if malloc fails ! */ + buffer.capacity = (start==NULL) ? 0 : bSize; + if (start==NULL) { + DEBUGLOG(5, "ZSTDMT_getBuffer: buffer allocation failure !!"); + } else { + DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize); + } + return buffer; + } +} + +#if ZSTD_RESIZE_SEQPOOL +/** ZSTDMT_resizeBuffer() : + * assumption : bufPool must be valid + * @return : a buffer that is at least the buffer pool buffer size. + * If a reallocation happens, the data in the input buffer is copied. + */ +static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) +{ + size_t const bSize = bufPool->bufferSize; + if (buffer.capacity < bSize) { + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer_t newBuffer; + newBuffer.start = start; + newBuffer.capacity = start == NULL ? 0 : bSize; + if (start != NULL) { + assert(newBuffer.capacity >= buffer.capacity); + ZSTD_memcpy(newBuffer.start, buffer.start, buffer.capacity); + DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); + return newBuffer; + } + DEBUGLOG(5, "ZSTDMT_resizeBuffer: buffer allocation failure !!"); + } + return buffer; +} +#endif + +/* store buffer for later re-use, up to pool capacity */ +static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) +{ + DEBUGLOG(5, "ZSTDMT_releaseBuffer"); + if (buf.start == NULL) return; /* compatible with release on NULL */ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers < bufPool->totalBuffers) { + bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u", + (U32)buf.capacity, (U32)(bufPool->nbBuffers-1)); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return; + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* Reached bufferPool capacity (should not happen) */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); + ZSTD_customFree(buf.start, bufPool->cMem); +} + +/* We need 2 output buffers per worker since each dstBuff must be flushed after it is released. + * The 3 additional buffers are as follows: + * 1 buffer for input loading + * 1 buffer for "next input" when submitting current one + * 1 buffer stuck in queue */ +#define BUF_POOL_MAX_NB_BUFFERS(nbWorkers) (2*(nbWorkers) + 3) + +/* After a worker releases its rawSeqStore, it is immediately ready for reuse. + * So we only need one seq buffer per worker. */ +#define SEQ_POOL_MAX_NB_BUFFERS(nbWorkers) (nbWorkers) + +/* ===== Seq Pool Wrapper ====== */ + +typedef ZSTDMT_bufferPool ZSTDMT_seqPool; + +static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) +{ + return ZSTDMT_sizeof_bufferPool(seqPool); +} + +static rawSeqStore_t bufferToSeq(buffer_t buffer) +{ + rawSeqStore_t seq = kNullRawSeqStore; + seq.seq = (rawSeq*)buffer.start; + seq.capacity = buffer.capacity / sizeof(rawSeq); + return seq; +} + +static buffer_t seqToBuffer(rawSeqStore_t seq) +{ + buffer_t buffer; + buffer.start = seq.seq; + buffer.capacity = seq.capacity * sizeof(rawSeq); + return buffer; +} + +static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) +{ + if (seqPool->bufferSize == 0) { + return kNullRawSeqStore; + } + return bufferToSeq(ZSTDMT_getBuffer(seqPool)); +} + +#if ZSTD_RESIZE_SEQPOOL +static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); +} +#endif + +static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); +} + +static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) +{ + ZSTDMT_setBufferSize(seqPool, nbSeq * sizeof(rawSeq)); +} + +static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(SEQ_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + if (seqPool == NULL) return NULL; + ZSTDMT_setNbSeq(seqPool, 0); + return seqPool; +} + +static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) +{ + ZSTDMT_freeBufferPool(seqPool); +} + +static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) +{ + return ZSTDMT_expandBufferPool(pool, SEQ_POOL_MAX_NB_BUFFERS(nbWorkers)); +} + + +/* ===== CCtx Pool ===== */ +/* a single CCtx Pool can be invoked from multiple threads in parallel */ + +typedef struct { + ZSTD_pthread_mutex_t poolMutex; + int totalCCtx; + int availCCtx; + ZSTD_customMem cMem; + ZSTD_CCtx* cctx[1]; /* variable size */ +} ZSTDMT_CCtxPool; + +/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ +static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) +{ + int cid; + for (cid=0; cidtotalCCtx; cid++) + ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ + ZSTD_pthread_mutex_destroy(&pool->poolMutex); + ZSTD_customFree(pool, pool->cMem); +} + +/* ZSTDMT_createCCtxPool() : + * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, + ZSTD_customMem cMem) +{ + ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_customCalloc( + sizeof(ZSTDMT_CCtxPool) + (nbWorkers-1)*sizeof(ZSTD_CCtx*), cMem); + assert(nbWorkers > 0); + if (!cctxPool) return NULL; + if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { + ZSTD_customFree(cctxPool, cMem); + return NULL; + } + cctxPool->cMem = cMem; + cctxPool->totalCCtx = nbWorkers; + cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ + cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } + DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); + return cctxPool; +} + +static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, + int nbWorkers) +{ + if (srcPool==NULL) return NULL; + if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ + /* need a larger cctx pool */ + { ZSTD_customMem const cMem = srcPool->cMem; + ZSTDMT_freeCCtxPool(srcPool); + return ZSTDMT_createCCtxPool(nbWorkers, cMem); + } +} + +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + { unsigned const nbWorkers = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool) + + (nbWorkers-1) * sizeof(ZSTD_CCtx*); + unsigned u; + size_t totalCCtxSize = 0; + for (u=0; ucctx[u]); + } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + assert(nbWorkers > 0); + return poolSize + totalCCtxSize; + } +} + +static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) +{ + DEBUGLOG(5, "ZSTDMT_getCCtx"); + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + if (cctxPool->availCCtx) { + cctxPool->availCCtx--; + { ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + return cctx; + } } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + DEBUGLOG(5, "create one more CCtx"); + return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */ +} + +static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return; /* compatibility with release on NULL */ + ZSTD_pthread_mutex_lock(&pool->poolMutex); + if (pool->availCCtx < pool->totalCCtx) + pool->cctx[pool->availCCtx++] = cctx; + else { + /* pool overflow : should not happen, since totalCCtx==nbWorkers */ + DEBUGLOG(4, "CCtx pool overflow : free cctx"); + ZSTD_freeCCtx(cctx); + } + ZSTD_pthread_mutex_unlock(&pool->poolMutex); +} + +/* ==== Serial State ==== */ + +typedef struct { + void const* start; + size_t size; +} range_t; + +typedef struct { + /* All variables in the struct are protected by mutex. */ + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + ZSTD_CCtx_params params; + ldmState_t ldmState; + XXH64_state_t xxhState; + unsigned nextJobID; + /* Protects ldmWindow. + * Must be acquired after the main mutex when acquiring both. + */ + ZSTD_pthread_mutex_t ldmWindowMutex; + ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */ + ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ +} serialState_t; + +static int +ZSTDMT_serialState_reset(serialState_t* serialState, + ZSTDMT_seqPool* seqPool, + ZSTD_CCtx_params params, + size_t jobSize, + const void* dict, size_t const dictSize, + ZSTD_dictContentType_e dictContentType) +{ + /* Adjust parameters */ + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashRateLog < 32); + } else { + ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + } + serialState->nextJobID = 0; + if (params.fParams.checksumFlag) + XXH64_reset(&serialState->xxhState, 0); + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_customMem cMem = params.customMem; + unsigned const hashLog = params.ldmParams.hashLog; + size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); + unsigned const bucketLog = + params.ldmParams.hashLog - params.ldmParams.bucketSizeLog; + unsigned const prevBucketLog = + serialState->params.ldmParams.hashLog - + serialState->params.ldmParams.bucketSizeLog; + size_t const numBuckets = (size_t)1 << bucketLog; + /* Size the seq pool tables */ + ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); + /* Reset the window */ + ZSTD_window_init(&serialState->ldmState.window); + /* Resize tables and output space if necessary. */ + if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(hashSize, cMem); + } + if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); + serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(numBuckets, cMem); + } + if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) + return 1; + /* Zero the tables */ + ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize); + ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets); + + /* Update window state and fill hash table with dict */ + serialState->ldmState.loadedDictEnd = 0; + if (dictSize > 0) { + if (dictContentType == ZSTD_dct_rawContent) { + BYTE const* const dictEnd = (const BYTE*)dict + dictSize; + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0); + ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); + serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); + } else { + /* don't even load anything */ + } + } + + /* Initialize serialState's copy of ldmWindow. */ + serialState->ldmWindow = serialState->ldmState.window; + } + + serialState->params = params; + serialState->params.jobSize = (U32)jobSize; + return 0; +} + +static int ZSTDMT_serialState_init(serialState_t* serialState) +{ + int initError = 0; + ZSTD_memset(serialState, 0, sizeof(*serialState)); + initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); + initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->ldmWindowCond, NULL); + return initError; +} + +static void ZSTDMT_serialState_free(serialState_t* serialState) +{ + ZSTD_customMem cMem = serialState->params.customMem; + ZSTD_pthread_mutex_destroy(&serialState->mutex); + ZSTD_pthread_cond_destroy(&serialState->cond); + ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); + ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); +} + +static void ZSTDMT_serialState_update(serialState_t* serialState, + ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore, + range_t src, unsigned jobID) +{ + /* Wait for our turn */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + while (serialState->nextJobID < jobID) { + DEBUGLOG(5, "wait for serialState->cond"); + ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex); + } + /* A future job may error and skip our job */ + if (serialState->nextJobID == jobID) { + /* It is now our turn, do any processing necessary */ + if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) { + size_t error; + assert(seqStore.seq != NULL && seqStore.pos == 0 && + seqStore.size == 0 && seqStore.capacity > 0); + assert(src.size <= serialState->params.jobSize); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0); + error = ZSTD_ldm_generateSequences( + &serialState->ldmState, &seqStore, + &serialState->params.ldmParams, src.start, src.size); + /* We provide a large enough buffer to never fail. */ + assert(!ZSTD_isError(error)); (void)error; + /* Update ldmWindow to match the ldmState.window and signal the main + * thread if it is waiting for a buffer. + */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + serialState->ldmWindow = serialState->ldmState.window; + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + if (serialState->params.fParams.checksumFlag && src.size > 0) + XXH64_update(&serialState->xxhState, src.start, src.size); + } + /* Now it is the next jobs turn */ + serialState->nextJobID++; + ZSTD_pthread_cond_broadcast(&serialState->cond); + ZSTD_pthread_mutex_unlock(&serialState->mutex); + + if (seqStore.size > 0) { + size_t const err = ZSTD_referenceExternalSequences( + jobCCtx, seqStore.seq, seqStore.size); + assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, + unsigned jobID, size_t cSize) +{ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + if (serialState->nextJobID <= jobID) { + assert(ZSTD_isError(cSize)); (void)cSize; + DEBUGLOG(5, "Skipping past job %u because of error", jobID); + serialState->nextJobID = jobID + 1; + ZSTD_pthread_cond_broadcast(&serialState->cond); + + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + ZSTD_window_clear(&serialState->ldmWindow); + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + ZSTD_pthread_mutex_unlock(&serialState->mutex); + +} + + +/* ------------------------------------------ */ +/* ===== Worker thread ===== */ +/* ------------------------------------------ */ + +static const range_t kNullRange = { NULL, 0 }; + +typedef struct { + size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ + size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ + ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ + ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ + ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ + serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */ + buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ + range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ + range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */ + unsigned jobID; /* set by mtctx, then read by worker => no barrier */ + unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ + unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ + ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ + const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ + unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ + size_t dstFlushed; /* used only by mtctx */ + unsigned frameChecksumNeeded; /* used only by mtctx */ +} ZSTDMT_jobDescription; + +#define JOB_ERROR(e) { \ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ + job->cSize = e; \ + ZSTD_pthread_mutex_unlock(&job->job_mutex); \ + goto _endJob; \ +} + +/* ZSTDMT_compressionJob() is a POOL_function type */ +static void ZSTDMT_compressionJob(void* jobDescription) +{ + ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; + ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ + ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); + rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); + buffer_t dstBuff = job->dstBuff; + size_t lastCBlockSize = 0; + + /* resources */ + if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation)); + if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ + dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation)); + job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */ + } + if (jobParams.ldmParams.enableLdm == ZSTD_ps_enable && rawSeqStore.seq == NULL) + JOB_ERROR(ERROR(memory_allocation)); + + /* Don't compute the checksum for chunks, since we compute it externally, + * but write it in the header. + */ + if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; + /* Don't run LDM for the chunks, since we handle it externally */ + jobParams.ldmParams.enableLdm = ZSTD_ps_disable; + /* Correct nbWorkers to 0. */ + jobParams.nbWorkers = 0; + + + /* init */ + if (job->cdict) { + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); + assert(job->firstJob); /* only allowed for first job */ + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } else { /* srcStart points at reloaded section */ + U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; + { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); + if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); + } + if (!job->firstJob) { + size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0); + if (ZSTD_isError(err)) JOB_ERROR(err); + } + { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, + job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ + ZSTD_dtlm_fast, + NULL, /*cdict*/ + &jobParams, pledgedSrcSize); + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } } + + /* Perform serial step as early as possible, but after CCtx initialization */ + ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); + + if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ + size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); + if (ZSTD_isError(hSize)) JOB_ERROR(hSize); + DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); + ZSTD_invalidateRepCodes(cctx); + } + + /* compress */ + { size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX; + int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize); + const BYTE* ip = (const BYTE*) job->src.start; + BYTE* const ostart = (BYTE*)dstBuff.start; + BYTE* op = ostart; + BYTE* oend = op + dstBuff.capacity; + int chunkNb; + if (sizeof(size_t) > sizeof(int)) assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* check overflow */ + DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); + assert(job->cSize == 0); + for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { + size_t const cSize = ZSTD_compressContinue_public(cctx, op, oend-op, ip, chunkSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + ip += chunkSize; + op += cSize; assert(op < oend); + /* stats */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + job->cSize += cSize; + job->consumed = chunkSize * chunkNb; + DEBUGLOG(5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)", + (U32)cSize, (U32)job->cSize); + ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */ + ZSTD_pthread_mutex_unlock(&job->job_mutex); + } + /* last block */ + assert(chunkSize > 0); + assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */ + if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) { + size_t const lastBlockSize1 = job->src.size & (chunkSize-1); + size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1; + size_t const cSize = (job->lastJob) ? + ZSTD_compressEnd_public(cctx, op, oend-op, ip, lastBlockSize) : + ZSTD_compressContinue_public(cctx, op, oend-op, ip, lastBlockSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + lastCBlockSize = cSize; + } } + if (!job->firstJob) { + /* Double check that we don't have an ext-dict, because then our + * repcode invalidation doesn't work. + */ + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); + } + ZSTD_CCtx_trace(cctx, 0); + +_endJob: + ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize); + if (job->prefix.size > 0) + DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); + DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); + /* release resources */ + ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); + ZSTDMT_releaseCCtx(job->cctxPool, cctx); + /* report */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0); + job->cSize += lastCBlockSize; + job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presumed completed */ + ZSTD_pthread_cond_signal(&job->job_cond); + ZSTD_pthread_mutex_unlock(&job->job_mutex); +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +typedef struct { + range_t prefix; /* read-only non-owned prefix buffer */ + buffer_t buffer; + size_t filled; +} inBuff_t; + +typedef struct { + BYTE* buffer; /* The round input buffer. All jobs get references + * to pieces of the buffer. ZSTDMT_tryGetInputRange() + * handles handing out job input buffers, and makes + * sure it doesn't overlap with any pieces still in use. + */ + size_t capacity; /* The capacity of buffer. */ + size_t pos; /* The position of the current inBuff in the round + * buffer. Updated past the end if the inBuff once + * the inBuff is sent to the worker thread. + * pos <= capacity. + */ +} roundBuff_t; + +static const roundBuff_t kNullRoundBuff = {NULL, 0, 0}; + +#define RSYNC_LENGTH 32 +/* Don't create chunks smaller than the zstd block size. + * This stops us from regressing compression ratio too much, + * and ensures our output fits in ZSTD_compressBound(). + * + * If this is shrunk < ZSTD_BLOCKSIZELOG_MIN then + * ZSTD_COMPRESSBOUND() will need to be updated. + */ +#define RSYNC_MIN_BLOCK_LOG ZSTD_BLOCKSIZELOG_MAX +#define RSYNC_MIN_BLOCK_SIZE (1< one job is already prepared, but pool has shortage of workers. Don't create a new job. */ + inBuff_t inBuff; + roundBuff_t roundBuff; + serialState_t serial; + rsyncState_t rsync; + unsigned jobIDMask; + unsigned doneJobID; + unsigned nextJobID; + unsigned frameEnded; + unsigned allJobsCompleted; + unsigned long long frameContentSize; + unsigned long long consumed; + unsigned long long produced; + ZSTD_customMem cMem; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + unsigned providedFactory: 1; +}; + +static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) +{ + U32 jobNb; + if (jobTable == NULL) return; + for (jobNb=0; jobNb mtctx->jobIDMask+1) { /* need more job capacity */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + mtctx->jobIDMask = 0; + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, mtctx->cMem); + if (mtctx->jobs==NULL) return ERROR(memory_allocation); + assert((nbJobs != 0) && ((nbJobs & (nbJobs - 1)) == 0)); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + } + return 0; +} + + +/* ZSTDMT_CCtxParam_setNbWorkers(): + * Internal use only */ +static size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) +{ + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); +} + +MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ + ZSTDMT_CCtx* mtctx; + U32 nbJobs = nbWorkers + 2; + int initError; + DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers); + + if (nbWorkers < 1) return NULL; + nbWorkers = MIN(nbWorkers , ZSTDMT_NBWORKERS_MAX); + if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL)) + /* invalid custom allocator */ + return NULL; + + mtctx = (ZSTDMT_CCtx*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtx), cMem); + if (!mtctx) return NULL; + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + mtctx->cMem = cMem; + mtctx->allJobsCompleted = 1; + if (pool != NULL) { + mtctx->factory = pool; + mtctx->providedFactory = 1; + } + else { + mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + mtctx->providedFactory = 0; + } + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); + assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + mtctx->bufPool = ZSTDMT_createBufferPool(BUF_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); + mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); + initError = ZSTDMT_serialState_init(&mtctx->serial); + mtctx->roundBuff = kNullRoundBuff; + if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | !mtctx->seqPool | initError) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + DEBUGLOG(3, "mt_cctx created, for %u threads", nbWorkers); + return mtctx; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool); +#else + (void)nbWorkers; + (void)cMem; + (void)pool; + return NULL; +#endif +} + + +/* ZSTDMT_releaseAllJobResources() : + * note : ensure all workers are killed first ! */ +static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) +{ + unsigned jobID; + DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); + for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { + /* Copy the mutex/cond out */ + ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex; + ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond; + + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); + + /* Clear the job description, but keep the mutex/cond */ + ZSTD_memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); + mtctx->jobs[jobID].job_mutex = mutex; + mtctx->jobs[jobID].job_cond = cond; + } + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->allJobsCompleted = 1; +} + +static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) +{ + DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); + while (mtctx->doneJobID < mtctx->nextJobID) { + unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask; + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); + while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { + DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */ + ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); + mtctx->doneJobID++; + } +} + +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx==NULL) return 0; /* compatible with free on NULL */ + if (!mtctx->providedFactory) + POOL_free(mtctx->factory); /* stop and free worker threads */ + ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + ZSTDMT_freeBufferPool(mtctx->bufPool); + ZSTDMT_freeCCtxPool(mtctx->cctxPool); + ZSTDMT_freeSeqPool(mtctx->seqPool); + ZSTDMT_serialState_free(&mtctx->serial); + ZSTD_freeCDict(mtctx->cdictLocal); + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + ZSTD_customFree(mtctx, mtctx->cMem); + return 0; +} + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + + POOL_sizeof(mtctx->factory) + + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) + + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTDMT_sizeof_seqPool(mtctx->seqPool) + + ZSTD_sizeof_CDict(mtctx->cdictLocal) + + mtctx->roundBuff.capacity; +} + + +/* ZSTDMT_resize() : + * @return : error code if fails, 0 on success */ +static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) +{ + if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , ""); + mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, BUF_POOL_MAX_NB_BUFFERS(nbWorkers)); + if (mtctx->bufPool == NULL) return ERROR(memory_allocation); + mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); + if (mtctx->cctxPool == NULL) return ERROR(memory_allocation); + mtctx->seqPool = ZSTDMT_expandSeqPool(mtctx->seqPool, nbWorkers); + if (mtctx->seqPool == NULL) return ERROR(memory_allocation); + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + return 0; +} + + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates a selected set of compression parameters, remaining compatible with currently active frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams) +{ + U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ + int const compressionLevel = cctxParams->compressionLevel; + DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", + compressionLevel); + mtctx->params.compressionLevel = compressionLevel; + { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + cParams.windowLog = saved_wlog; + mtctx->params.cParams = cParams; + } +} + +/* ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + * Note : mutex will be acquired during statistics collection inside workers. */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) +{ + ZSTD_frameProgression fps; + DEBUGLOG(5, "ZSTDMT_getFrameProgression"); + fps.ingested = mtctx->consumed + mtctx->inBuff.filled; + fps.consumed = mtctx->consumed; + fps.produced = fps.flushed = mtctx->produced; + fps.currentJobID = mtctx->nextJobID; + fps.nbActiveWorkers = 0; + { unsigned jobNb; + unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); + DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", + mtctx->doneJobID, lastJobNb, mtctx->jobReady) + for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) { + unsigned const wJobID = jobNb & mtctx->jobIDMask; + ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + fps.ingested += jobPtr->src.size; + fps.consumed += jobPtr->consumed; + fps.produced += produced; + fps.flushed += flushed; + fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + } + return fps; +} + + +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) +{ + size_t toFlush; + unsigned const jobID = mtctx->doneJobID; + assert(jobID <= mtctx->nextJobID); + if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */ + + /* look into oldest non-fully-flushed job */ + { unsigned const wJobID = jobID & mtctx->jobIDMask; + ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + assert(jobPtr->consumed <= jobPtr->src.size); + toFlush = produced - flushed; + /* if toFlush==0, nothing is available to flush. + * However, jobID is expected to still be active: + * if jobID was already completed and fully flushed, + * ZSTDMT_flushProduced() should have already moved onto next job. + * Therefore, some input has not yet been consumed. */ + if (toFlush==0) { + assert(jobPtr->consumed < jobPtr->src.size); + } + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + + return toFlush; +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) +{ + unsigned jobLog; + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on cycleLog instead. */ + jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3); + } else { + jobLog = MAX(20, params->cParams.windowLog + 2); + } + return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); +} + +static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) +{ + switch(strat) + { + case ZSTD_btultra2: + return 9; + case ZSTD_btultra: + case ZSTD_btopt: + return 8; + case ZSTD_btlazy2: + case ZSTD_lazy2: + return 7; + case ZSTD_lazy: + case ZSTD_greedy: + case ZSTD_dfast: + case ZSTD_fast: + default:; + } + return 6; +} + +static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) +{ + assert(0 <= ovlog && ovlog <= 9); + if (ovlog == 0) return ZSTDMT_overlapLog_default(strat); + return ovlog; +} + +static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) +{ + int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); + assert(0 <= overlapRLog && overlapRLog <= 8); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. + * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ + ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + - overlapRLog; + } + assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); + DEBUGLOG(4, "overlapLog : %i", params->overlapLog); + DEBUGLOG(4, "overlap size : %i", 1 << ovLog); + return (ovLog==0) ? 0 : (size_t)1 << ovLog; +} + +/* ====================================== */ +/* ======= Streaming API ======= */ +/* ====================================== */ + +size_t ZSTDMT_initCStream_internal( + ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", + (U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx); + + /* params supposed partially fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + /* init */ + if (params.nbWorkers != mtctx->params.nbWorkers) + FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , ""); + + if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; + + DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); + + if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + mtctx->allJobsCompleted = 1; + } + + mtctx->params = params; + mtctx->frameContentSize = pledgedSrcSize; + if (dict) { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = NULL; + mtctx->cdict = cdict; + } + + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); + DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); + mtctx->targetSectionSize = params.jobSize; + if (mtctx->targetSectionSize == 0) { + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); + } + assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); + + if (params.rsyncable) { + /* Aim for the targetsectionSize as the average job size. */ + U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10); + U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10); + /* We refuse to create jobs < RSYNC_MIN_BLOCK_SIZE bytes, so make sure our + * expected job size is at least 4x larger. */ + assert(rsyncBits >= RSYNC_MIN_BLOCK_LOG + 2); + DEBUGLOG(4, "rsyncLog = %u", rsyncBits); + mtctx->rsync.hash = 0; + mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; + mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); + } + if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ + DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize); + DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); + ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); + { + /* If ldm is enabled we need windowSize space. */ + size_t const windowSize = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable ? (1U << mtctx->params.cParams.windowLog) : 0; + /* Two buffers of slack, plus extra space for the overlap + * This is the minimum slack that LDM works with. One extra because + * flush might waste up to targetSectionSize-1 bytes. Another extra + * for the overlap (if > 0), then one to fill which doesn't overlap + * with the LDM window. + */ + size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0); + size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers; + /* Compute the total size, and always have enough slack */ + size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1); + size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers; + size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; + if (mtctx->roundBuff.capacity < capacity) { + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem); + if (mtctx->roundBuff.buffer == NULL) { + mtctx->roundBuff.capacity = 0; + return ERROR(memory_allocation); + } + mtctx->roundBuff.capacity = capacity; + } + } + DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity>>10)); + mtctx->roundBuff.pos = 0; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->inBuff.prefix = kNullRange; + mtctx->doneJobID = 0; + mtctx->nextJobID = 0; + mtctx->frameEnded = 0; + mtctx->allJobsCompleted = 0; + mtctx->consumed = 0; + mtctx->produced = 0; + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, + dict, dictSize, dictContentType)) + return ERROR(memory_allocation); + return 0; +} + + +/* ZSTDMT_writeLastEmptyBlock() + * Write a single empty block with an end-of-frame to finish a frame. + * Job must be created from streaming variant. + * This function is always successful if expected conditions are fulfilled. + */ +static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) +{ + assert(job->lastJob == 1); + assert(job->src.size == 0); /* last job is empty -> will be simplified into a last empty block */ + assert(job->firstJob == 0); /* cannot be first job, as it also needs to create frame header */ + assert(job->dstBuff.start == NULL); /* invoked from streaming variant only (otherwise, dstBuff might be user's output) */ + job->dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (job->dstBuff.start == NULL) { + job->cSize = ERROR(memory_allocation); + return; + } + assert(job->dstBuff.capacity >= ZSTD_blockHeaderSize); /* no buffer should ever be that small */ + job->src = kNullRange; + job->cSize = ZSTD_writeLastEmptyBlock(job->dstBuff.start, job->dstBuff.capacity); + assert(!ZSTD_isError(job->cSize)); + assert(job->consumed == 0); +} + +static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZSTD_EndDirective endOp) +{ + unsigned const jobID = mtctx->nextJobID & mtctx->jobIDMask; + int const endFrame = (endOp == ZSTD_e_end); + + if (mtctx->nextJobID > mtctx->doneJobID + mtctx->jobIDMask) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: will not create new job : table is full"); + assert((mtctx->nextJobID & mtctx->jobIDMask) == (mtctx->doneJobID & mtctx->jobIDMask)); + return 0; + } + + if (!mtctx->jobReady) { + BYTE const* src = (BYTE const*)mtctx->inBuff.buffer.start; + DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ", + mtctx->nextJobID, (U32)srcSize, (U32)mtctx->inBuff.prefix.size); + mtctx->jobs[jobID].src.start = src; + mtctx->jobs[jobID].src.size = srcSize; + assert(mtctx->inBuff.filled >= srcSize); + mtctx->jobs[jobID].prefix = mtctx->inBuff.prefix; + mtctx->jobs[jobID].consumed = 0; + mtctx->jobs[jobID].cSize = 0; + mtctx->jobs[jobID].params = mtctx->params; + mtctx->jobs[jobID].cdict = mtctx->nextJobID==0 ? mtctx->cdict : NULL; + mtctx->jobs[jobID].fullFrameSize = mtctx->frameContentSize; + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + mtctx->jobs[jobID].cctxPool = mtctx->cctxPool; + mtctx->jobs[jobID].bufPool = mtctx->bufPool; + mtctx->jobs[jobID].seqPool = mtctx->seqPool; + mtctx->jobs[jobID].serial = &mtctx->serial; + mtctx->jobs[jobID].jobID = mtctx->nextJobID; + mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0); + mtctx->jobs[jobID].lastJob = endFrame; + mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0); + mtctx->jobs[jobID].dstFlushed = 0; + + /* Update the round buffer pos and clear the input buffer to be reset */ + mtctx->roundBuff.pos += srcSize; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + /* Set the prefix */ + if (!endFrame) { + size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize); + mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize; + mtctx->inBuff.prefix.size = newPrefixSize; + } else { /* endFrame==1 => no need for another input buffer */ + mtctx->inBuff.prefix = kNullRange; + mtctx->frameEnded = endFrame; + if (mtctx->nextJobID == 0) { + /* single job exception : checksum is already calculated directly within worker thread */ + mtctx->params.fParams.checksumFlag = 0; + } } + + if ( (srcSize == 0) + && (mtctx->nextJobID>0)/*single job must also write frame header*/ ) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: creating a last empty block to end frame"); + assert(endOp == ZSTD_e_end); /* only possible case : need to end the frame with an empty last block */ + ZSTDMT_writeLastEmptyBlock(mtctx->jobs + jobID); + mtctx->nextJobID++; + return 0; + } + } + + DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u, jobNb == %u (mod:%u))", + mtctx->nextJobID, + (U32)mtctx->jobs[jobID].src.size, + mtctx->jobs[jobID].lastJob, + mtctx->nextJobID, + jobID); + if (POOL_tryAdd(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[jobID])) { + mtctx->nextJobID++; + mtctx->jobReady = 0; + } else { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: no worker available for job %u", mtctx->nextJobID); + mtctx->jobReady = 1; + } + return 0; +} + + +/*! ZSTDMT_flushProduced() : + * flush whatever data has been produced but not yet flushed in current job. + * move to next job if current one is fully flushed. + * `output` : `pos` will be updated with amount of data flushed . + * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . + * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ +static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned blockToFlush, ZSTD_EndDirective end) +{ + unsigned const wJobID = mtctx->doneJobID & mtctx->jobIDMask; + DEBUGLOG(5, "ZSTDMT_flushProduced (blocking:%u , job %u <= %u)", + blockToFlush, mtctx->doneJobID, mtctx->nextJobID); + assert(output->size >= output->pos); + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + if ( blockToFlush + && (mtctx->doneJobID < mtctx->nextJobID) ) { + assert(mtctx->jobs[wJobID].dstFlushed <= mtctx->jobs[wJobID].cSize); + while (mtctx->jobs[wJobID].dstFlushed == mtctx->jobs[wJobID].cSize) { /* nothing to flush */ + if (mtctx->jobs[wJobID].consumed == mtctx->jobs[wJobID].src.size) { + DEBUGLOG(5, "job %u is completely consumed (%u == %u) => don't wait for cond, there will be none", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].consumed, (U32)mtctx->jobs[wJobID].src.size); + break; + } + DEBUGLOG(5, "waiting for something to flush from job %u (currently flushed: %u bytes)", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTD_pthread_cond_wait(&mtctx->jobs[wJobID].job_cond, &mtctx->jobs[wJobID].job_mutex); /* block when nothing to flush but some to come */ + } } + + /* try to flush something */ + { size_t cSize = mtctx->jobs[wJobID].cSize; /* shared */ + size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */ + size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */ + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + if (ZSTD_isError(cSize)) { + DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s", + mtctx->doneJobID, ZSTD_getErrorName(cSize)); + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + return cSize; + } + /* add frame checksum if necessary (can only happen once) */ + assert(srcConsumed <= srcSize); + if ( (srcConsumed == srcSize) /* job completed -> worker no longer active */ + && mtctx->jobs[wJobID].frameChecksumNeeded ) { + U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); + DEBUGLOG(4, "ZSTDMT_flushProduced: writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].cSize, checksum); + cSize += 4; + mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ + mtctx->jobs[wJobID].frameChecksumNeeded = 0; + } + + if (cSize > 0) { /* compression is ongoing or completed */ + size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); + DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", + (U32)toFlush, mtctx->doneJobID, (U32)srcConsumed, (U32)srcSize, (U32)cSize); + assert(mtctx->doneJobID < mtctx->nextJobID); + assert(cSize >= mtctx->jobs[wJobID].dstFlushed); + assert(mtctx->jobs[wJobID].dstBuff.start != NULL); + if (toFlush > 0) { + ZSTD_memcpy((char*)output->dst + output->pos, + (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, + toFlush); + } + output->pos += toFlush; + mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ + + if ( (srcConsumed == srcSize) /* job is completed */ + && (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ + DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff); + DEBUGLOG(5, "dstBuffer released"); + mtctx->jobs[wJobID].dstBuff = g_nullBuffer; + mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */ + mtctx->consumed += srcSize; + mtctx->produced += cSize; + mtctx->doneJobID++; + } } + + /* return value : how many bytes left in buffer ; fake it to 1 when unknown but >0 */ + if (cSize > mtctx->jobs[wJobID].dstFlushed) return (cSize - mtctx->jobs[wJobID].dstFlushed); + if (srcSize > srcConsumed) return 1; /* current job not completely compressed */ + } + if (mtctx->doneJobID < mtctx->nextJobID) return 1; /* some more jobs ongoing */ + if (mtctx->jobReady) return 1; /* one job is ready to push, just not yet in the list */ + if (mtctx->inBuff.filled > 0) return 1; /* input is not empty, and still needs to be converted into a job */ + mtctx->allJobsCompleted = mtctx->frameEnded; /* all jobs are entirely flushed => if this one is last one, frame is completed */ + if (end == ZSTD_e_end) return !mtctx->frameEnded; /* for ZSTD_e_end, question becomes : is frame completed ? instead of : are internal buffers fully flushed ? */ + return 0; /* internal buffers fully flushed */ +} + +/** + * Returns the range of data used by the earliest job that is not yet complete. + * If the data of the first job is broken up into two segments, we cover both + * sections. + */ +static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) +{ + unsigned const firstJobID = mtctx->doneJobID; + unsigned const lastJobID = mtctx->nextJobID; + unsigned jobID; + + for (jobID = firstJobID; jobID < lastJobID; ++jobID) { + unsigned const wJobID = jobID & mtctx->jobIDMask; + size_t consumed; + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + consumed = mtctx->jobs[wJobID].consumed; + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + + if (consumed < mtctx->jobs[wJobID].src.size) { + range_t range = mtctx->jobs[wJobID].prefix; + if (range.size == 0) { + /* Empty prefix */ + range = mtctx->jobs[wJobID].src; + } + /* Job source in multiple segments not supported yet */ + assert(range.start <= mtctx->jobs[wJobID].src.start); + return range; + } + } + return kNullRange; +} + +/** + * Returns non-zero iff buffer and range overlap. + */ +static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) +{ + BYTE const* const bufferStart = (BYTE const*)buffer.start; + BYTE const* const rangeStart = (BYTE const*)range.start; + + if (rangeStart == NULL || bufferStart == NULL) + return 0; + + { + BYTE const* const bufferEnd = bufferStart + buffer.capacity; + BYTE const* const rangeEnd = rangeStart + range.size; + + /* Empty ranges cannot overlap */ + if (bufferStart == bufferEnd || rangeStart == rangeEnd) + return 0; + + return bufferStart < rangeEnd && rangeStart < bufferEnd; + } +} + +static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) +{ + range_t extDict; + range_t prefix; + + DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); + extDict.start = window.dictBase + window.lowLimit; + extDict.size = window.dictLimit - window.lowLimit; + + prefix.start = window.base + window.dictLimit; + prefix.size = window.nextSrc - (window.base + window.dictLimit); + DEBUGLOG(5, "extDict [0x%zx, 0x%zx)", + (size_t)extDict.start, + (size_t)extDict.start + extDict.size); + DEBUGLOG(5, "prefix [0x%zx, 0x%zx)", + (size_t)prefix.start, + (size_t)prefix.start + prefix.size); + + return ZSTDMT_isOverlapped(buffer, extDict) + || ZSTDMT_isOverlapped(buffer, prefix); +} + +static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) +{ + if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; + DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); + DEBUGLOG(5, "source [0x%zx, 0x%zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + ZSTD_PTHREAD_MUTEX_LOCK(mutex); + while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { + DEBUGLOG(5, "Waiting for LDM to finish..."); + ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); + } + DEBUGLOG(6, "Done waiting for LDM to finish"); + ZSTD_pthread_mutex_unlock(mutex); + } +} + +/** + * Attempts to set the inBuff to the next section to fill. + * If any part of the new section is still in use we give up. + * Returns non-zero if the buffer is filled. + */ +static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) +{ + range_t const inUse = ZSTDMT_getInputDataInUse(mtctx); + size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos; + size_t const target = mtctx->targetSectionSize; + buffer_t buffer; + + DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); + assert(mtctx->inBuff.buffer.start == NULL); + assert(mtctx->roundBuff.capacity >= target); + + if (spaceLeft < target) { + /* ZSTD_invalidateRepCodes() doesn't work for extDict variants. + * Simply copy the prefix to the beginning in that case. + */ + BYTE* const start = (BYTE*)mtctx->roundBuff.buffer; + size_t const prefixSize = mtctx->inBuff.prefix.size; + + buffer.start = start; + buffer.capacity = prefixSize; + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + ZSTDMT_waitForLdmComplete(mtctx, buffer); + ZSTD_memmove(start, mtctx->inBuff.prefix.start, prefixSize); + mtctx->inBuff.prefix.start = start; + mtctx->roundBuff.pos = prefixSize; + } + buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos; + buffer.capacity = target; + + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); + + ZSTDMT_waitForLdmComplete(mtctx, buffer); + + DEBUGLOG(5, "Using prefix range [%zx, %zx)", + (size_t)mtctx->inBuff.prefix.start, + (size_t)mtctx->inBuff.prefix.start + mtctx->inBuff.prefix.size); + DEBUGLOG(5, "Using source range [%zx, %zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + + + mtctx->inBuff.buffer = buffer; + mtctx->inBuff.filled = 0; + assert(mtctx->roundBuff.pos + buffer.capacity <= mtctx->roundBuff.capacity); + return 1; +} + +typedef struct { + size_t toLoad; /* The number of bytes to load from the input. */ + int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ +} syncPoint_t; + +/** + * Searches through the input for a synchronization point. If one is found, we + * will instruct the caller to flush, and return the number of bytes to load. + * Otherwise, we will load as many bytes as possible and instruct the caller + * to continue as normal. + */ +static syncPoint_t +findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) +{ + BYTE const* const istart = (BYTE const*)input.src + input.pos; + U64 const primePower = mtctx->rsync.primePower; + U64 const hitMask = mtctx->rsync.hitMask; + + syncPoint_t syncPoint; + U64 hash; + BYTE const* prev; + size_t pos; + + syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled); + syncPoint.flush = 0; + if (!mtctx->params.rsyncable) + /* Rsync is disabled. */ + return syncPoint; + if (mtctx->inBuff.filled + input.size - input.pos < RSYNC_MIN_BLOCK_SIZE) + /* We don't emit synchronization points if it would produce too small blocks. + * We don't have enough input to find a synchronization point, so don't look. + */ + return syncPoint; + if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) + /* Not enough to compute the hash. + * We will miss any synchronization points in this RSYNC_LENGTH byte + * window. However, since it depends only in the internal buffers, if the + * state is already synchronized, we will remain synchronized. + * Additionally, the probability that we miss a synchronization point is + * low: RSYNC_LENGTH / targetSectionSize. + */ + return syncPoint; + /* Initialize the loop variables. */ + if (mtctx->inBuff.filled < RSYNC_MIN_BLOCK_SIZE) { + /* We don't need to scan the first RSYNC_MIN_BLOCK_SIZE positions + * because they can't possibly be a sync point. So we can start + * part way through the input buffer. + */ + pos = RSYNC_MIN_BLOCK_SIZE - mtctx->inBuff.filled; + if (pos >= RSYNC_LENGTH) { + prev = istart + pos - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + } else { + assert(mtctx->inBuff.filled >= RSYNC_LENGTH); + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev + pos, (RSYNC_LENGTH - pos)); + hash = ZSTD_rollingHash_append(hash, istart, pos); + } + } else { + /* We have enough bytes buffered to initialize the hash, + * and have processed enough bytes to find a sync point. + * Start scanning at the beginning of the input. + */ + assert(mtctx->inBuff.filled >= RSYNC_MIN_BLOCK_SIZE); + assert(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH); + pos = 0; + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + if ((hash & hitMask) == hitMask) { + /* We're already at a sync point so don't load any more until + * we're able to flush this sync point. + * This likely happened because the job table was full so we + * couldn't add our job. + */ + syncPoint.toLoad = 0; + syncPoint.flush = 1; + return syncPoint; + } + } + /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll + * through the input. If we hit a synchronization point, then cut the + * job off, and tell the compressor to flush the job. Otherwise, load + * all the bytes and continue as normal. + * If we go too long without a synchronization point (targetSectionSize) + * then a block will be emitted anyways, but this is okay, since if we + * are already synchronized we will remain synchronized. + */ + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + for (; pos < syncPoint.toLoad; ++pos) { + BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; + /* This assert is very expensive, and Debian compiles with asserts enabled. + * So disable it for now. We can get similar coverage by checking it at the + * beginning & end of the loop. + * assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + */ + hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); + assert(mtctx->inBuff.filled + pos >= RSYNC_MIN_BLOCK_SIZE); + if ((hash & hitMask) == hitMask) { + syncPoint.toLoad = pos + 1; + syncPoint.flush = 1; + ++pos; /* for assert */ + break; + } + } + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + return syncPoint; +} + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx) +{ + size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled; + if (hintInSize==0) hintInSize = mtctx->targetSectionSize; + return hintInSize; +} + +/** ZSTDMT_compressStream_generic() : + * internal use only - exposed to be invoked from zstd_compress.c + * assumption : output and input are valid (pos <= size) + * @return : minimum amount of data remaining to flush, 0 if none */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + unsigned forwardInputProgress = 0; + DEBUGLOG(5, "ZSTDMT_compressStream_generic (endOp=%u, srcSize=%u)", + (U32)endOp, (U32)(input->size - input->pos)); + assert(output->pos <= output->size); + assert(input->pos <= input->size); + + if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { + /* current frame being ended. Only flush/end are allowed */ + return ERROR(stage_wrong); + } + + /* fill input buffer */ + if ( (!mtctx->jobReady) + && (input->size > input->pos) ) { /* support NULL input */ + if (mtctx->inBuff.buffer.start == NULL) { + assert(mtctx->inBuff.filled == 0); /* Can't fill an empty buffer */ + if (!ZSTDMT_tryGetInputRange(mtctx)) { + /* It is only possible for this operation to fail if there are + * still compression jobs ongoing. + */ + DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed"); + assert(mtctx->doneJobID != mtctx->nextJobID); + } else + DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); + } + if (mtctx->inBuff.buffer.start != NULL) { + syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input); + if (syncPoint.flush && endOp == ZSTD_e_continue) { + endOp = ZSTD_e_flush; + } + assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); + DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", + (U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); + ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); + input->pos += syncPoint.toLoad; + mtctx->inBuff.filled += syncPoint.toLoad; + forwardInputProgress = syncPoint.toLoad>0; + } + } + if ((input->pos < input->size) && (endOp == ZSTD_e_end)) { + /* Can't end yet because the input is not fully consumed. + * We are in one of these cases: + * - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job. + * - We filled the input buffer: flush this job but don't end the frame. + * - We hit a synchronization point: flush this job but don't end the frame. + */ + assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable); + endOp = ZSTD_e_flush; + } + + if ( (mtctx->jobReady) + || (mtctx->inBuff.filled >= mtctx->targetSectionSize) /* filled enough : let's compress */ + || ((endOp != ZSTD_e_continue) && (mtctx->inBuff.filled > 0)) /* something to flush : let's go */ + || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */ + size_t const jobSize = mtctx->inBuff.filled; + assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , ""); + } + + /* check for potential compressed data ready to be flushed */ + { size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */ + if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */ + DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush); + return remainingToFlush; + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.h b/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.h new file mode 100644 index 000000000..ed4dc0e99 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + #ifndef ZSTDMT_COMPRESS_H + #define ZSTDMT_COMPRESS_H + + #if defined (__cplusplus) + extern "C" { + #endif + + +/* Note : This is an internal API. + * These APIs used to be exposed with ZSTDLIB_API, + * because it used to be the only way to invoke MT compression. + * Now, you must use ZSTD_compress2 and ZSTD_compressStream2() instead. + * + * This API requires ZSTD_MULTITHREAD to be defined during compilation, + * otherwise ZSTDMT_createCCtx*() will fail. + */ + +/* === Dependencies === */ +#include "../common/zstd_deps.h" /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ + + +/* === Constants === */ +#ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */ +# define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256) +#endif +#ifndef ZSTDMT_JOBSIZE_MIN /* a different value can be selected at compile time */ +# define ZSTDMT_JOBSIZE_MIN (512 KB) +#endif +#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) + + +/* ======================================================== + * === Private interface, for use by ZSTD_compress.c === + * === Not exposed in libzstd. Never invoke directly === + * ======================================================== */ + +/* === Memory management === */ +typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; +/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, + ZSTD_customMem cMem, + ZSTD_threadPool *pool); +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); + +/* === Streaming functions === */ + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * mtctx can be freshly constructed or reused from a prior compression. + * If mtctx is reused, memory allocations from the prior compression may not be freed, + * even if they are not needed for the current compression. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + +/*! ZSTDMT_compressStream_generic() : + * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() + * depending on flush directive. + * @return : minimum amount of data still to be flushed + * 0 if fully flushed + * or an error code + * note : needs to be init using any ZSTD_initCStream*() variant */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + /*! ZSTDMT_toFlushNow() + * Tell how many bytes are ready to be flushed immediately. + * Probe the oldest active job (not yet entirely flushed) and check its output buffer. + * If return 0, it means there is no active job, + * or, it means oldest job is still active, but everything produced has been flushed so far, + * therefore flushing is limited by speed of oldest job. */ +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates only a selected set of compression parameters, to remain compatible with current frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); + +/*! ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTDMT_COMPRESS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress.c b/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress.c new file mode 100644 index 000000000..5b217ac58 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress.c @@ -0,0 +1,1882 @@ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" /* BIT_* */ +#include "../common/fse.h" /* to compress headers */ +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/zstd_internal.h" +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */ + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 + +/* ************************************************************** +* Macros +****************************************************************/ + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_FAST_BMI2_ATTRS +#endif + +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + if (flags & HUF_flags_bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + (void)flags; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilimit [in] - The input limit, stop when any input pointer is below ilimit. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilimit, but that indicates corruption. + */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilimit; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); + +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const ilimit = (const BYTE*)src + 6 + 8; + + BYTE* const oend = (BYTE*)dst + dstSize; + + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; + + /* Read the jump table. */ + { + const BYTE* const istart = (const BYTE*)src; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + * length1 must be >= 16 so that ip[0] >= ilimit before the loop + * starts. + */ + if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* If ip[] >= ilimit, it is guaranteed to be safe to + * reload bits[]. It may be beyond its section, but is + * guaranteed to be valid (>= istart). + */ + args->ilimit = ilimit; + + args->oend = oend; + args->dt = dt; + + return 1; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->iend[0]; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ + +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (U64)((symbol << 8) + nbBits); + } else { + D4 = (U64)(symbol + (nbBits << 8)); + } + assert(D4 < (1U << 16)); + D4 *= 0x0001000100010001ULL; + return D4; +} + +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { int n; + U32 nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return (size_t)(pEnd-pStart); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilimit = args->ilimit; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + int symbol; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilimit); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilimit) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we get close to the end. */ + if (op[3] + 20 > olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + + do { + /* Decode 5 symbols in each of the 4 streams */ + for (symbol = 0; symbol < 5; ++symbol) { + for (stream = 0; stream < 4; ++stream) { + int const index = (int)(bits[stream] >> 53); + int const entry = (int)dtable[index]; + bits[stream] <<= (entry & 63); + op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF); + } + } + /* Reload the bitstreams */ + for (stream = 0; stream < 4; ++stream) { + int const ctz = ZSTD_countTrailingZeros64(bits[stream]); + int const nbBits = ctz & 7; + int const nbBytes = ctz >> 3; + op[stream] += 5; + ip[stream] -= nbBytes; + bits[stream] = MEM_read64(ip[stream]) | 1; + bits[stream] <<= nbBits; + } + } while (op[3] < olimit); + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilimit); + loopFn(&args); + + /* Our loop guarantees that ip[] >= ilimit and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) + +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +#endif + + if (!(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilimit = args->ilimit; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + int symbol; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilimit); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilimit) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + + /* Exit the fast decoding loop if we are too close to the end. */ + if (op[3] + 10 > olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + + do { + /* Do 5 table lookups for each of the first 3 streams */ + for (symbol = 0; symbol < 5; ++symbol) { + for (stream = 0; stream < 3; ++stream) { + int const index = (int)(bits[stream] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[stream], entry.sequence); + bits[stream] <<= (entry.nbBits); + op[stream] += (entry.length); + } + } + /* Do 1 table lookup from the final stream */ + { + int const index = (int)(bits[3] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[3], entry.sequence); + bits[3] <<= (entry.nbBits); + op[3] += (entry.length); + } + /* Do 4 table lookups from the final stream & reload bitstreams */ + for (stream = 0; stream < 4; ++stream) { + /* Do a table lookup from the final stream. + * This is interleaved with the reloading to reduce register + * pressure. This shouldn't be necessary, but compilers can + * struggle with codegen with high register pressure. + */ + { + int const index = (int)(bits[3] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[3], entry.sequence); + bits[3] <<= (entry.nbBits); + op[3] += (entry.length); + } + /* Reload the bistreams. The final bitstream must be reloaded + * after the 5th symbol was decoded. + */ + { + int const ctz = ZSTD_countTrailingZeros64(bits[stream]); + int const nbBits = ctz & 7; + int const nbBytes = ctz >> 3; + ip[stream] -= nbBytes; + bits[stream] = MEM_read64(ip[stream]) | 1; + bits[stream] <<= nbBits; + } + } + } while (op[3] < olimit); + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilimit); + loopFn(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} + +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif + + if (!(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); +} + +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} +#endif + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#endif + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress_amd64.S b/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress_amd64.S new file mode 100644 index 000000000..671624fe3 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress_amd64.S @@ -0,0 +1,576 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "../common/portability_macros.h" + +/* Stack marking + * ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart + */ +#if defined(__ELF__) && defined(__GNUC__) +.section .note.GNU-stack,"",%progbits +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +/* Calling convention: + * + * %rdi contains the first argument: HUF_DecompressAsmArgs*. + * %rbp isn't maintained (no frame pointer). + * %rsp contains the stack pointer that grows down. + * No red-zone is assumed, only addresses >= %rsp are used. + * All register contents are preserved. + * + * TODO: Support Windows calling convention. + */ + +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +.global HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.text + +/* Sets up register mappings for clarity. + * op[], bits[], dtable & ip[0] each get their own register. + * ip[1,2,3] & olimit alias var[]. + * %rax is a scratch register. + */ + +#define op0 rsi +#define op1 rbx +#define op2 rcx +#define op3 rdi + +#define ip0 r8 +#define ip1 r9 +#define ip2 r10 +#define ip3 r11 + +#define bits0 rbp +#define bits1 rdx +#define bits2 r12 +#define bits3 r13 +#define dtable r14 +#define olimit r15 + +/* var[] aliases ip[1,2,3] & olimit + * ip[1,2,3] are saved every iteration. + * olimit is only used in compute_olimit. + */ +#define var0 r15 +#define var1 r9 +#define var2 r10 +#define var3 r11 + +/* 32-bit var registers */ +#define vard0 r15d +#define vard1 r9d +#define vard2 r10d +#define vard3 r11d + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM(X) \ + X(0); \ + X(1); \ + X(2); \ + X(3) + +/* Calls X(N, idx) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM_WITH_INDEX(X, idx) \ + X(0, idx); \ + X(1, idx); \ + X(2, idx); \ + X(3, idx) + +/* Define both _HUF_* & HUF_* symbols because MacOS + * C symbols are prefixed with '_' & Linux symbols aren't. + */ +_HUF_decompress4X1_usingDTable_internal_fast_asm_loop: +HUF_decompress4X1_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + /* Read HUF_DecompressAsmArgs* args from %rax */ + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push 104(%rax) /* ilimit */ + push 112(%rax) /* oend */ + push %olimit /* olimit space */ + + subq $24, %rsp + +.L_4X1_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rbx, rdx must be saved + * op3 & ip0 mustn't be clobbered + */ + movq %rbx, 0(%rsp) + movq %rdx, 8(%rsp) + + movq 32(%rsp), %rax /* rax = oend */ + subq %op3, %rax /* rax = oend - op3 */ + + /* r15 = (oend - op3) / 5 */ + movabsq $-3689348814741910323, %rdx + mulq %rdx + movq %rdx, %r15 + shrq $2, %r15 + + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilimit */ + subq %rdx, %rax /* rax = ip0 - ilimit */ + movq %rax, %rbx /* rbx = ip0 - ilimit */ + + /* rdx = (ip0 - ilimit) / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %rbx + shrq %rbx + addq %rbx, %rdx + shrq $2, %rdx + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* r15 = r15 * 5 */ + leaq (%r15, %r15, 4), %r15 + + /* olimit = op3 + r15 */ + addq %op3, %olimit + + movq 8(%rsp), %rdx + movq 0(%rsp), %rbx + + /* If (op3 + 20 > olimit) */ + movq %op3, %rax /* rax = op3 */ + addq $20, %rax /* rax = op3 + 20 */ + cmpq %rax, %olimit /* op3 + 20 > olimit */ + jb .L_4X1_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X1_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X1_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X1_exit + +/* Reads top 11 bits from bits[n] + * Loads dt[bits[n]] into var[n] + */ +#define GET_NEXT_DELT(n) \ + movq $53, %var##n; \ + shrxq %var##n, %bits##n, %var##n; \ + movzwl (%dtable,%var##n,2),%vard##n + +/* var[n] must contain the DTable entry computed with GET_NEXT_DELT + * Moves var[n] to %rax + * bits[n] <<= var[n] & 63 + * op[n][idx] = %rax >> 8 + * %ah is a way to access bits [8, 16) of %rax + */ +#define DECODE_FROM_DELT(n, idx) \ + movq %var##n, %rax; \ + shlxq %var##n, %bits##n, %bits##n; \ + movb %ah, idx(%op##n) + +/* Assumes GET_NEXT_DELT has been called. + * Calls DECODE_FROM_DELT then GET_NEXT_DELT + */ +#define DECODE_AND_GET_NEXT(n, idx) \ + DECODE_FROM_DELT(n, idx); \ + GET_NEXT_DELT(n) \ + +/* // ctz & nbBytes is stored in bits[n] + * // nbBits is stored in %rax + * ctz = CTZ[bits[n]] + * nbBits = ctz & 7 + * nbBytes = ctz >> 3 + * op[n] += 5 + * ip[n] -= nbBytes + * // Note: x86-64 is little-endian ==> no bswap + * bits[n] = MEM_readST(ip[n]) | 1 + * bits[n] <<= nbBits + */ +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + andq $7, %rax; \ + shrq $3, %bits##n; \ + leaq 5(%op##n), %op##n; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlx %rax, %bits##n, %bits##n + + /* Store clobbered variables on the stack */ + movq %olimit, 24(%rsp) + movq %ip1, 0(%rsp) + movq %ip2, 8(%rsp) + movq %ip3, 16(%rsp) + + /* Call GET_NEXT_DELT for each stream */ + FOR_EACH_STREAM(GET_NEXT_DELT) + + .p2align 6 + +.L_4X1_loop_body: + /* Decode 5 symbols in each of the 4 streams (20 total) + * Must have called GET_NEXT_DELT for each stream + */ + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE_FROM_DELT, 4) + + /* Load ip[1,2,3] from stack (var[] aliases them) + * ip[] is needed for RELOAD_BITS + * Each will be stored back to the stack after RELOAD + */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Reload each stream & fetch the next table entry + * to prepare for the next iteration + */ + RELOAD_BITS(0) + GET_NEXT_DELT(0) + + RELOAD_BITS(1) + movq %ip1, 0(%rsp) + GET_NEXT_DELT(1) + + RELOAD_BITS(2) + movq %ip2, 8(%rsp) + GET_NEXT_DELT(2) + + RELOAD_BITS(3) + movq %ip3, 16(%rsp) + GET_NEXT_DELT(3) + + /* If op3 < olimit: continue the loop */ + cmp %op3, 24(%rsp) + ja .L_4X1_loop_body + + /* Reload ip[1,2,3] from stack */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Re-compute olimit */ + jmp .L_4X1_compute_olimit + +#undef GET_NEXT_DELT +#undef DECODE_FROM_DELT +#undef DECODE +#undef RELOAD_BITS +.L_4X1_exit: + addq $24, %rsp + + /* Restore stack (oend & olimit) */ + pop %rax /* olimit */ + pop %rax /* oend */ + pop %rax /* ilimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +_HUF_decompress4X2_usingDTable_internal_fast_asm_loop: +HUF_decompress4X2_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push %rax /* olimit */ + push 104(%rax) /* ilimit */ + + movq 112(%rax), %rax + push %rax /* oend3 */ + + movq %op3, %rax + push %rax /* oend2 */ + + movq %op2, %rax + push %rax /* oend1 */ + + movq %op1, %rax + push %rax /* oend0 */ + + /* Scratch space */ + subq $8, %rsp + +.L_4X2_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rdx must be saved + * op[1,2,3,4] & ip0 mustn't be clobbered + */ + movq %rdx, 0(%rsp) + + /* We can consume up to 7 input bytes each iteration. */ + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilimit */ + subq %rdx, %rax /* rax = ip0 - ilimit */ + movq %rax, %r15 /* r15 = ip0 - ilimit */ + + /* rdx = rax / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %r15 + shrq %r15 + addq %r15, %rdx + shrq $2, %rdx + + /* r15 = (ip0 - ilimit) / 7 */ + movq %rdx, %r15 + + /* r15 = min(r15, min(oend0 - op0, oend1 - op1, oend2 - op2, oend3 - op3) / 10) */ + movq 8(%rsp), %rax /* rax = oend0 */ + subq %op0, %rax /* rax = oend0 - op0 */ + movq 16(%rsp), %rdx /* rdx = oend1 */ + subq %op1, %rdx /* rdx = oend1 - op1 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movq 24(%rsp), %rax /* rax = oend2 */ + subq %op2, %rax /* rax = oend2 - op2 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movq 32(%rsp), %rax /* rax = oend3 */ + subq %op3, %rax /* rax = oend3 - op3 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movabsq $-3689348814741910323, %rax + mulq %rdx + shrq $3, %rdx /* rdx = rdx / 10 */ + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* olimit = op3 + 5 * r15 */ + movq %r15, %rax + leaq (%op3, %rax, 4), %olimit + addq %rax, %olimit + + movq 0(%rsp), %rdx + + /* If (op3 + 10 > olimit) */ + movq %op3, %rax /* rax = op3 */ + addq $10, %rax /* rax = op3 + 10 */ + cmpq %rax, %olimit /* op3 + 10 > olimit */ + jb .L_4X2_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X2_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X2_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X2_exit + +#define DECODE(n, idx) \ + movq %bits##n, %rax; \ + shrq $53, %rax; \ + movzwl 0(%dtable,%rax,4),%r8d; \ + movzbl 2(%dtable,%rax,4),%r15d; \ + movzbl 3(%dtable,%rax,4),%eax; \ + movw %r8w, (%op##n); \ + shlxq %r15, %bits##n, %bits##n; \ + addq %rax, %op##n + +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + shrq $3, %bits##n; \ + andq $7, %rax; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlxq %rax, %bits##n, %bits##n + + + movq %olimit, 48(%rsp) + + .p2align 6 + +.L_4X2_loop_body: + /* We clobber r8, so store it on the stack */ + movq %r8, 0(%rsp) + + /* Decode 5 symbols from each of the 4 streams (20 symbols total). */ + FOR_EACH_STREAM_WITH_INDEX(DECODE, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 4) + + /* Reload r8 */ + movq 0(%rsp), %r8 + + FOR_EACH_STREAM(RELOAD_BITS) + + cmp %op3, 48(%rsp) + ja .L_4X2_loop_body + jmp .L_4X2_compute_olimit + +#undef DECODE +#undef RELOAD_BITS +.L_4X2_exit: + addq $8, %rsp + /* Restore stack (oend & olimit) */ + pop %rax /* oend0 */ + pop %rax /* oend1 */ + pop %rax /* oend2 */ + pop %rax /* oend3 */ + pop %rax /* ilimit */ + pop %rax /* olimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.c b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.c new file mode 100644 index 000000000..309ec0d03 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted, ""); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ddict->dictID; +} diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.h b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.h new file mode 100644 index 000000000..c4ca8877a --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + + +#endif /* ZSTD_DDICT_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress.c b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress.c new file mode 100644 index 000000000..7bc271342 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress.c @@ -0,0 +1,2355 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ +#include "../common/zstd_internal.h" /* blockProperties_t */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ +#include "../common/bits.h" /* ZSTD_highbit32 */ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/************************************* + * Multiple DDicts Hashset internals * + *************************************/ + +#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ + +#define DDICT_HASHSET_TABLE_BASE_SIZE 64 +#define DDICT_HASHSET_RESIZE_FACTOR 2 + +/* Hash function to determine starting position of dict insertion within the table + * Returns an index between [0, hashSet->ddictPtrTableSize] + */ +static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) { + const U64 hash = XXH64(&dictID, sizeof(U32), 0); + /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */ + return hash & (hashSet->ddictPtrTableSize - 1); +} + +/* Adds DDict to a hashset without resizing it. + * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set. + * Returns 0 if successful, or a zstd error code if something went wrong. + */ +static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) { + const U32 dictID = ZSTD_getDictID_fromDDict(ddict); + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!"); + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + while (hashSet->ddictPtrTable[idx] != NULL) { + /* Replace existing ddict if inserting ddict with same dictID */ + if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) { + DEBUGLOG(4, "DictID already exists, replacing rather than adding"); + hashSet->ddictPtrTable[idx] = ddict; + return 0; + } + idx &= idxRangeMask; + idx++; + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + hashSet->ddictPtrTable[idx] = ddict; + hashSet->ddictPtrCount++; + return 0; +} + +/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and + * rehashes all values, allocates new table, frees old table. + * Returns 0 on success, otherwise a zstd error code. + */ +static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR; + const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem); + const ZSTD_DDict** oldTable = hashSet->ddictPtrTable; + size_t oldTableSize = hashSet->ddictPtrTableSize; + size_t i; + + DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize); + RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!"); + hashSet->ddictPtrTable = newTable; + hashSet->ddictPtrTableSize = newTableSize; + hashSet->ddictPtrCount = 0; + for (i = 0; i < oldTableSize; ++i) { + if (oldTable[i] != NULL) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), ""); + } + } + ZSTD_customFree((void*)oldTable, customMem); + DEBUGLOG(4, "Finished re-hash"); + return 0; +} + +/* Fetches a DDict with the given dictID + * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL. + */ +static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) { + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + for (;;) { + size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]); + if (currDictID == dictID || currDictID == 0) { + /* currDictID == 0 implies a NULL ddict entry */ + break; + } else { + idx &= idxRangeMask; /* Goes to start of table when we reach the end */ + idx++; + } + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + return hashSet->ddictPtrTable[idx]; +} + +/* Allocates space for and returns a ddict hash set + * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with. + * Returns NULL if allocation failed. + */ +static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { + ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); + DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; + ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); + return NULL; + } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; + return ret; +} + +/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself. + * Note: The ZSTD_DDict* within the table are NOT freed. + */ +static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + DEBUGLOG(4, "Freeing ddict hash set"); + if (hashSet && hashSet->ddictPtrTable) { + ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem); + } + if (hashSet) { + ZSTD_customFree(hashSet, customMem); + } +} + +/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set. + * Returns 0 on success, or a ZSTD error. + */ +static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) { + DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize); + if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), ""); + } + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), ""); + return 0; +} + +/*-************************************************************* +* Context management +***************************************************************/ +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) + + ZSTD_sizeof_DDict(dctx->ddictLocal) + + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->outBufferMode = ZSTD_bm_buffered; + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; + dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->staticSize = 0; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; +#endif + dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif + dctx->ddictSet = NULL; + ZSTD_DCtx_resetParameters(dctx); +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; +} + +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + dctx->customMem = customMem; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = dctx->customMem; + ZSTD_clearDict(dctx); + ZSTD_customFree(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif + if (dctx->ddictSet) { + ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); + dctx->ddictSet = NULL; + } + ZSTD_customFree(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on + * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then + * accordingly sets the ddict to be used to decompress the frame. + * + * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is. + * + * ZSTD_d_refMultipleDDicts must be enabled for this function to be called. + */ +static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) { + assert(dctx->refMultipleDDicts && dctx->ddictSet); + DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame"); + if (dctx->ddict) { + const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID); + if (frameDDict) { + DEBUGLOG(4, "DDict found!"); + ZSTD_clearDict(dctx); + dctx->dictID = dctx->fParams.dictID; + dctx->ddict = frameDDict; + dctx->dictUses = ZSTD_use_indefinitely; + } + } +} + + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + +/** ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + RETURN_ERROR(prefix_unknown, ""); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported, ""); + { size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); + return skippableSize; + } +} + +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + +/** ZSTD_findDecompressedSize() : + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; + + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; + } + /* skip to next frame */ + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + + +/** ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * If multiple DDict references are enabled, also will choose the correct DDict to use. + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) return result; /* invalid header */ + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); + + /* Reference DDict requested by frame if dctx references multiple ddicts */ + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { + ZSTD_DCtx_selectFrameDDict(dctx); + } + +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), + dictionary_wrong, ""); +#endif + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); + dctx->processedCSize += headerSize; + return 0; +} + +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} + +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +{ + ZSTD_frameSizeInfo frameSizeInfo; + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameSizeInfoLegacy(src, srcSize); +#endif + + if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); + return frameSizeInfo; + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + size_t nbBlocks = 0; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Iterate over each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; + + if (blockProperties.lastBlock) break; + } + + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + ip += 4; + } + + frameSizeInfo.nbBlocks = nbBlocks; + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : (unsigned long long)nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; + } +} + +/** ZSTD_findFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + return frameSizeInfo.compressedSize; +} + +/** ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippeable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} + +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_frameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +/** ZSTD_insertBlock() : + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); + ZSTD_checkContinuity(dctx, blockStart, blockSize); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (srcSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memmove(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (regenSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memset(dst, b, regenSize); + return regenSize; +} + +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +{ +#if ZSTD_TRACE + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + if (dctx->ddict) { + trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict); + trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict); + trace.dictionaryIsCold = dctx->ddictIsCold; + } + trace.uncompressedSize = (size_t)uncompressedSize; + trace.compressedSize = (size_t)compressedSize; + trace.dctx = dctx; + ZSTD_trace_decompress_end(dctx->traceCtx, &trace); + } +#else + (void)dctx; + (void)uncompressedSize; + (void)compressedSize; + (void)streaming; +#endif +} + + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* const istart = (const BYTE*)(*srcPtr); + const BYTE* ip = istart; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + BYTE* oBlockEnd = oend; + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, /* frame */ 1, not_streaming); + break; + case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->validateChecksum) + XXH64_update(&dctx->xxhState, op, decodedSize); + if (decodedSize != 0) + op += decodedSize; + assert(ip != NULL); + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected, ""); + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } + ip += 4; + remainingSrcSize -= 4; + } + ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); + /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr); + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return (size_t)(op-ostart); +} + +static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) return frameSize; + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, + "legacy support is not compatible with static dctx"); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + + assert(decodedSize <= dstCapacity); + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + if (srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; /* check next frame */ + } } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); + } + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); + if (ZSTD_isError(res)) return res; + assert(res <= dstCapacity); + if (res != 0) + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); + + return (size_t)((BYTE*)dst - (BYTE*)dststart); +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + ZSTD_FALLTHROUGH; + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +/** + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return BOUNDED(1, inputSize, dctx->expected); +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + ZSTD_FALLTHROUGH; + case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + dctx->processedCSize += srcSize; + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_raw : + assert(srcSize <= dctx->expected); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_reserved : /* should never happen */ + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(rSize, ""); + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected, ""); + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize, /* flags */ 0); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, + dictPtr, (size_t)(dictEnd - dictPtr), + workspace, workspaceSize, /* flags */ 0); +#endif + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, + dictionary_corrupted, ""); + entropy->rep[i] = rep; + } } + + return (size_t)(dictPtr - (const BYTE*)dict); +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); +#if ZSTD_TRACE + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; +#endif + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->processedCSize = 0; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + dctx->bType = bt_reserved; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (dict && dictSize) + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted, ""); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", + dctx->ddictIsCold ? "~cold~" : "hot!"); + } + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompress frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (dict && dictSize != 0) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; + } + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); + dctx->dictUses = ZSTD_use_once; + return 0; +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); + return ZSTD_startingInputLength(zds->format); +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); + return ZSTD_startingInputLength(dctx->format); +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + DEBUGLOG(4, "ZSTD_resetDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); + return ZSTD_startingInputLength(dctx->format); +} + + +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) { + if (dctx->ddictSet == NULL) { + dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem); + if (!dctx->ddictSet) { + RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!"); + } + } + assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */ + FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), ""); + } + } + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; + return bounds; + case ZSTD_d_refMultipleDDicts: + bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; + bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; + return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ +} + +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + case ZSTD_d_refMultipleDDicts: + *value = (int)dctx->refMultipleDDicts; + return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + switch(dParam) { + case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; + return 0; + case ZSTD_d_refMultipleDDicts: + CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value); + if (dctx->staticSize != 0) { + RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!"); + } + dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; + return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + ZSTD_DCtx_resetParameters(dctx); + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/ + unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge, ""); + return minRBSize; +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge, ""); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + + +/* ***** Decompression ***** */ + +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_bm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); +} + +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (zds->outBufferMode == ZSTD_bm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(zds->outBufferMode == ZSTD_bm_stable); + } + return 0; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; + const char* ip = istart; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + DEBUGLOG(5, "stage zdss_init => transparent reset "); + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + zds->legacyVersion = 0; +#endif + zds->hostageByte = 0; + zds->expectedOutBuffer = *output; + ZSTD_FALLTHROUGH; + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) { + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; + return hint; + } } +#endif + { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + if (zds->refMultipleDDicts && zds->ddictSet) { + ZSTD_DCtx_selectFrameDDict(zds); + } + if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); + const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; + size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; + DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, + zds->previousLegacyVersion, legacyVersion, + dict, dictSize), ""); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ + return hint; + } } +#endif + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend-ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + if (cSize <= (size_t)(iend-istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + assert(istart != NULL); + ip = istart + cSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (zds->outBufferMode == ZSTD_bm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); + + if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >>10), + (U32)(zds->maxWindowSize >> 10) ); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge, ""); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_customFree(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } + zds->streamStage = zdss_read; + ZSTD_FALLTHROUGH; + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); + ip += neededInSize; + /* Function modifies the stage so we must break */ + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + ZSTD_FALLTHROUGH; + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend-ip)); + } else { + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); + } + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } + case zdss_flush: + { + size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); + + op = op ? op + flushedSize : op; + + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + + if ((ip==istart) && (op==ostart)) { /* no forward progress */ + zds->noForwardProgress ++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} + +size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.c b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.c new file mode 100644 index 000000000..09896a931 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.c @@ -0,0 +1,2192 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/compiler.h" /* prefetch */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" +#include "../common/bits.h" /* ZSTD_highbit32 */ + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); + + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); + return cSize; + } +} + +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) + { + /* room for litbuffer to fit without read faulting */ + dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } + else if (litSize > ZSTD_LITBUFFEREXTRASIZE) + { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } + else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + } + else + { + /* fits entirely within litExtraBuffer, so no split is necessary */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } +} + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity, const streaming_operation streaming); +/*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) +{ + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); + ZSTD_FALLTHROUGH; + + case set_compressed: + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } else { + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); + } + } + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + } + + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + RETURN_ERROR(corruption_detected, "impossible"); + } + } +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what empirically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].baseValue = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold)) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u=0; u max, corruption_detected, ""); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); + *DTablePtr = DTableSpace; + return headerSize; + } + default : + assert(0); + RETURN_ERROR(GENERIC, "impossible"); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); + + /* SeqHead */ + nbSeq = *ip++; + if (!nbSeq) { + *nbSeqPtr=0; + RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); + return 1; + } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; + } else { + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + /* FSE table descriptors */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; +} seqState_t; + +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op += oend_w - op; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) +{ + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offsets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +{ + seq_t seq; + /* + * ZSTD_seqSymbol is a structure with a total of 64 bits wide. So it can be + * loaded in one operation and extracted its fields by simply shifting or + * bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. + */ +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + } + + return seq; +} + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} + +MEM_STATIC void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ +#if DEBUGLEVEL >= 1 + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif +} +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer"); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + + /* decompress without overrunning litPtr begins */ + { + seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif +#endif + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) { + size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + } + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (--nbSeq) + BIT_reloadDStream(&(seqState.DStream)); + } + } + } + + if (nbSeq > 0) /* there is remaining lit from extra buffer */ + { + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for (; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + { + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE size_t +ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 8 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS STORED_SEQS + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + + dctx->fseEntropy = 1; + { int i; for (i=0; ientropy.rep[i]; } + assert(dst != NULL); + assert(iend >= ip); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) + { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + } + RETURN_ERROR_IF(seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) + { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */ + { + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} + +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + } + + return info; +} + +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } +} + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame, const streaming_operation streaming) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX. + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX)); + size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); + + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } + } + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } +} + + +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) +{ + if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.h b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.h new file mode 100644 index 000000000..9d1318882 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* DCtx, and some public functions */ +#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame, const streaming_operation streaming); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); + +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_internal.h b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_internal.h new file mode 100644 index 000000000..c2ec5d9fb --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_internal.h @@ -0,0 +1,238 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/mem.h" /* BYTE, U16, U32 */ +#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; + +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; +#if DYNAMIC_BMI2 != 0 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; +#endif + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; +#endif +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 != 0 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/zdict.h b/External/Zstd/zstd-1.5.5/lib/zdict.h new file mode 100644 index 000000000..2268f948a --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/zdict.h @@ -0,0 +1,474 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_ZDICT_H +#define ZSTD_ZDICT_H + +/*====== Dependencies ======*/ +#include /* size_t */ + + +/* ===== ZDICTLIB_API : control library symbols visibility ===== */ +#ifndef ZDICTLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZDICTLIB_VISIBLE +# endif +#endif + +#ifndef ZDICTLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZDICTLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZDICTLIB_API ZDICTLIB_VISIBLE +#endif + +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetition in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select the smallest size that doesn't + * hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictionary + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + + +/*! ZDICT_trainFromBuffer(): + * Train a dictionary from an array of samples. + * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4, + * f=20, and accel=1. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Note: Dictionary training will fail if there are not enough samples to construct a + * dictionary, or if most of the samples are too small (< 8 bytes being the lower limit). + * If dictionary training fails, you should use zstd without a dictionary, as the dictionary + * would've been ineffective anyways. If you believe your samples would benefit from a dictionary + * please open an issue with details, and we can look into it. + * Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); + +typedef struct { + int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ +} ZDICT_params_t; + +/*! ZDICT_finalizeDictionary(): + * Given a custom content as a basis for dictionary, and a set of samples, + * finalize dictionary by adding headers and statistics according to the zstd + * dictionary format. + * + * Samples must be stored concatenated in a flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each + * sample in order. The samples are used to construct the statistics, so they + * should be representative of what you will compress with this dictionary. + * + * The compression level can be set in `parameters`. You should pass the + * compression level you expect to use in production. The statistics for each + * compression level differ, so tuning the dictionary for the compression level + * can help quite a bit. + * + * You can set an explicit dictionary ID in `parameters`, or allow us to pick + * a random dictionary ID for you, but we can't guarantee no collisions. + * + * The dstDictBuffer and the dictContent may overlap, and the content will be + * appended to the end of the header. If the header + the content doesn't fit in + * maxDictSize the beginning of the content is truncated to make room, since it + * is presumed that the most profitable content is at the end of the dictionary, + * since that is the cheapest to reference. + * + * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). + * + * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), + * or an error code, which can be tested by ZDICT_isError(). + * Note: ZDICT_finalizeDictionary() will push notifications into stderr if + * instructed to, using notificationLevel>0. + * NOTE: This function currently may fail in several edge cases including: + * * Not enough samples + * * Samples are uncompressible + * * Samples are all exactly the same + */ +ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize, + const void* dictContent, size_t dictContentSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + + +/*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); + +#endif /* ZSTD_ZDICT_H */ + +#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC) +#define ZSTD_ZDICT_H_STATIC + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZDICTLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE +# else +# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE +# endif +#endif + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +#define ZDICT_DICTSIZE_MIN 256 +/* Deprecated: Remove in v1.6.0 */ +#define ZDICT_CONTENTSIZE_MIN 128 + +/*! ZDICT_cover_params_t: + * k and d are the only required parameters. + * For others, value 0 means default. + */ +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + ZDICT_params_t zParams; +} ZDICT_cover_params_t; + +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ + unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + + ZDICT_params_t zParams; +} ZDICT_fastCover_params_t; + +/*! ZDICT_trainFromBuffer_cover(): + * Train a dictionary from an array of samples using the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, + const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_cover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations and picks the best parameters. + * `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * + * All of the parameters d, k, steps are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. + */ +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t* parameters); + +/*! ZDICT_trainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * d and k are required. + * All other parameters are optional, will use default values if not provided + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, + size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_fastCover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations (specifically, k and d combinations) + * and picks the best parameters. `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * All of the parameters d, k, steps, f, and accel are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * If f is zero, default value of 20 is used. + * If accel is zero, default value of 1 is used. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. + */ +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, + size_t dictBufferCapacity, const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t* parameters); + +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ + ZDICT_params_t zParams; +} ZDICT_legacy_params_t; + +/*! ZDICT_trainFromBuffer_legacy(): + * Train a dictionary from an array of samples. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * `parameters` is optional and can be provided with values set to 0 to mean "default". + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t parameters); + + +/* Deprecation warnings */ +/* It is generally possible to disable deprecation warnings from compiler, + for example with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ +#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS +# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */ +#else +# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] +# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (ZDICT_GCC_VERSION >= 301) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZDICT_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") +# define ZDICT_DEPRECATED(message) +# endif +#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ + +ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +ZDICTLIB_STATIC_API +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + +#endif /* ZSTD_ZDICT_H_STATIC */ + +#if defined (__cplusplus) +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/zstd.h b/External/Zstd/zstd-1.5.5/lib/zstd.h new file mode 100644 index 000000000..e5c3f8b68 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/zstd.h @@ -0,0 +1,3020 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependencies ======*/ +#include /* INT_MAX */ +#include /* size_t */ + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 5 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +/* ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()` or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a context just once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to behave similarly to `ZSTD_compress()`, + * this function compresses at requested compression level, + * __ignoring any other parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters. + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove from experimental API entry points which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_useBlockSplitter + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will re-use the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-used multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* But if `output.pos == output.size`, there might be some data left within internal buffers., +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder finished and flushed all remaining buffers. + * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers, + * call ZSTD_decompressStream() again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + */ +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is re-used, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final + * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable + * or ZSTD_ps_disable allow for a force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_paramSwitch_e; + +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_frameHeader; + +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + +/*! ZSTD_generateSequences() : + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2(). + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters + * @return : number of sequences generated + */ + +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences( ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsSize, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * for any compression level up to selected one. + * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate + * does not include space for a window buffer. + * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * When srcSize can be bound by a known and rather "small" value, + * this fact can be used to provide a tighter estimation + * because the CCtx compression context will need less memory. + * This tighter estimation can be provided by more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * + * Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_DStream memory budget depends on window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + * Note 2 : only single-threaded compression is supported. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size (default:0) */ +#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if conditions are not respected. + * + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to rely on user provided buffer instead. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_useBlockSplitter + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + * + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_searchForExternalRepcodes + * This parameter affects how zstd parses external sequences, such as sequences + * provided through the compressSequences() API or from an external block-level + * sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets in + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level. + * + * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future. + */ +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * re-use decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t ZSTD_sequenceProducer_F ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F* sequenceProducer +); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ + +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ + +/*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/zstd_errors.h b/External/Zstd/zstd-1.5.5/lib/zstd_errors.h new file mode 100644 index 000000000..dc75eeeba --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/zstd_errors.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stabilityCondition_notRespected = 50, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ From a1c70608a0c6fa018eb5951a8fa134aad6247a4d Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Jul 2023 13:42:47 +0930 Subject: [PATCH 058/129] ThreadPool: Create a general purpose ThreadPool - Will allow operations outside of the JobQueue to be parallelized - Set thread names for profiling automatically - Fix unsafe threadname setting in debug builds (pointer to string with unsafe lifetime) --- Code/Core/CoreTest/TestMain.cpp | 1 + Code/Core/CoreTest/Tests/TestThreadPool.cpp | 89 ++++++++++ Code/Core/Process/Thread.cpp | 29 ++-- Code/Core/Process/ThreadPool.cpp | 173 ++++++++++++++++++++ Code/Core/Process/ThreadPool.h | 70 ++++++++ 5 files changed, 352 insertions(+), 10 deletions(-) create mode 100644 Code/Core/CoreTest/Tests/TestThreadPool.cpp create mode 100644 Code/Core/Process/ThreadPool.cpp create mode 100644 Code/Core/Process/ThreadPool.h diff --git a/Code/Core/CoreTest/TestMain.cpp b/Code/Core/CoreTest/TestMain.cpp index 670a32d01..21062fc1a 100644 --- a/Code/Core/CoreTest/TestMain.cpp +++ b/Code/Core/CoreTest/TestMain.cpp @@ -29,6 +29,7 @@ int main( int, char *[] ) REGISTER_TESTGROUP( TestSystemMutex ) REGISTER_TESTGROUP( TestTestTCPConnectionPool ) REGISTER_TESTGROUP( TestThread ) + REGISTER_TESTGROUP( TestThreadPool ) REGISTER_TESTGROUP( TestTimer ) REGISTER_TESTGROUP( TestUnorderedMap ) diff --git a/Code/Core/CoreTest/Tests/TestThreadPool.cpp b/Code/Core/CoreTest/Tests/TestThreadPool.cpp new file mode 100644 index 000000000..0e72ae6f3 --- /dev/null +++ b/Code/Core/CoreTest/Tests/TestThreadPool.cpp @@ -0,0 +1,89 @@ +// TestThreadPool.cpp +//------------------------------------------------------------------------------ + +// Includes +//------------------------------------------------------------------------------ +// TestFramework +#include "TestFramework/TestGroup.h" + +// Core +#include "Core/Process/Atomic.h" +#include "Core/Process/ThreadPool.h" + +// TestThread +//------------------------------------------------------------------------------ +class TestThreadPool : public TestGroup +{ +private: + DECLARE_TESTS + + // Tests + void Unused() const; + void SingleJob() const; + void MultipleJobs() const; + + // Helpers + static void Increment( void * userData ) + { + Atomic * u32 = static_cast *>( userData ); + u32->Increment(); + } +}; + +// Register Tests +//------------------------------------------------------------------------------ +REGISTER_TESTS_BEGIN( TestThreadPool ) + REGISTER_TEST( Unused ) + REGISTER_TEST( SingleJob ) + REGISTER_TEST( MultipleJobs ) +REGISTER_TESTS_END + +// Unused +//------------------------------------------------------------------------------ +void TestThreadPool::Unused() const +{ + // A thread pool never used + ThreadPool threadPool( 4 ); +} + +// SingleJob +//------------------------------------------------------------------------------ +void TestThreadPool::SingleJob() const +{ + Atomic count; + + // Create + ThreadPool * threadPool = FNEW( ThreadPool( 4 ) ); + + // Enqueue jobs + threadPool->EnqueueJob( Increment, &count ); + + // Destroy + FDELETE threadPool; + + TEST_ASSERT( count.Load() == 1 ); +} + +// MultipleJobs +//------------------------------------------------------------------------------ +void TestThreadPool::MultipleJobs() const +{ + Atomic count; + + // Create + ThreadPool * threadPool = FNEW( ThreadPool( 4 ) ); + + // Enqueue jobs + const uint32_t numJobs = 1024; + for ( uint32_t i = 0; i < numJobs; ++i ) + { + threadPool->EnqueueJob( Increment, &count ); + } + + // Destroy + FDELETE threadPool; + + TEST_ASSERT( count.Load() == numJobs ); +} + +//------------------------------------------------------------------------------ diff --git a/Code/Core/Process/Thread.cpp b/Code/Core/Process/Thread.cpp index a07a00ee7..d014bf7e2 100644 --- a/Code/Core/Process/Thread.cpp +++ b/Code/Core/Process/Thread.cpp @@ -48,12 +48,16 @@ struct ThreadStartInfo ThreadStartInfo( Thread::ThreadEntryFunction entryFunc, void * userData, const char * threadName ) : m_UserEntryFunction( entryFunc ) , m_UserData( userData ) - , m_ThreadName( threadName ) - {} + { + if ( threadName ) + { + m_ThreadName = threadName; + } + } Thread::ThreadEntryFunction m_UserEntryFunction; void * m_UserData; - const char * m_ThreadName; + AString m_ThreadName; #if defined( __WINDOWS__ ) static uint32_t WINAPI ThreadStartFunction( void * userData ) @@ -66,13 +70,16 @@ struct ThreadStartInfo Thread::ThreadEntryFunction realFunction = originalInfo->m_UserEntryFunction; void * realUserData = originalInfo->m_UserData; - // Set thread name for debugging - #ifdef DEBUG - if ( originalInfo->m_ThreadName ) - { - Thread::SetThreadName( originalInfo->m_ThreadName ); - } - #endif + if ( originalInfo->m_ThreadName.IsEmpty() == false ) + { + // Set thread name for debugging + #ifdef DEBUG + Thread::SetThreadName( originalInfo->m_ThreadName.Get() ); + #endif + + // Set thread name for profiling + PROFILE_SET_THREAD_NAME( originalInfo->m_ThreadName.Get() ); + } // done with ThreadStartInfo MemoryBarrier(); @@ -85,6 +92,8 @@ struct ThreadStartInfo return (void *)(size_t)( (*realFunction)( realUserData ) ); #endif } + + void operator =(const ThreadStartInfo& other) = delete; }; // Static Data diff --git a/Code/Core/Process/ThreadPool.cpp b/Code/Core/Process/ThreadPool.cpp new file mode 100644 index 000000000..907aaa189 --- /dev/null +++ b/Code/Core/Process/ThreadPool.cpp @@ -0,0 +1,173 @@ +// ThreadPool +//------------------------------------------------------------------------------ + +// Includes +//------------------------------------------------------------------------------ +#include "ThreadPool.h" + +// Core +#include "Core/Process/Thread.h" +#include "Core/Profile/Profile.h" +#include "Core/Strings/AStackString.h" + +// CONSTRUCTOR +//------------------------------------------------------------------------------ +ThreadPool::ThreadPool( uint32_t numThreads ) + : m_NumThreads( numThreads ) + #if defined( __WINDOWS__ ) + , m_WakeSemaphore( numThreads ) // On Windows, take advantage of signalling limit + #else + , m_WakeSemaphore() + #endif +{ + PROFILE_FUNCTION; + + // Doesn't make sense to create a pool with no threads + ASSERT( m_NumThreads > 0 ); + + // Start first thread (it will create additional threads) + const uint32_t threadId = 1; + m_FirstThread = FNEW( ThreadPoolThread( threadId, *this ) ); +} + +// DESTRUCTOR +//------------------------------------------------------------------------------ +ThreadPool::~ThreadPool() +{ + PROFILE_FUNCTION; + + // Signal worker threads to exit + m_WantToQuit.Store( true ); + m_WakeSemaphore.Signal( m_NumThreads ); + + // Destroy first thread (it will destroy additional threads) + FDELETE m_FirstThread; + + // All jobs should have been completed + ASSERT( m_JobQueue.IsEmpty() ); +} + +// EnqueueJob +//------------------------------------------------------------------------------ +void ThreadPool::EnqueueJob( ThreadJobFunc func, void * userData ) +{ + // Add job to queue + { + MutexHolder mh( m_JobQueueMutex ); + ThreadJob & job = m_JobQueue.EmplaceBack(); + job.m_Function = func; + job.m_UserData = userData; + } + + // Wake threads if sleeping + m_WakeSemaphore.Signal(); +} + +// ThreadFuncWrapper +//------------------------------------------------------------------------------ +/*static*/ uint32_t ThreadPool::ThreadFuncWrapper( void * userData ) +{ + ThreadPoolThread * threadPoolThread = static_cast( userData ); + threadPoolThread->ThreadFunc(); + return 0; +} + +// ThreadFunc +//------------------------------------------------------------------------------ +void ThreadPool::ThreadFunc() +{ + for ( ;; ) + { + // See if a job is available + // (Jobs might be taken by other threads even if we were woken) + ThreadJob job; + { + MutexHolder mh( m_JobQueueMutex ); + if ( m_JobQueue.IsEmpty() == false ) + { + // Remove Job from queue, processing them in order of enqueuing + // Popping from the front of an Array is inefficient, but current + // uses have few jobs (typically one per-core) + job = m_JobQueue[ 0 ]; + m_JobQueue.PopFront(); + } + } + + // Job available? + if ( job.m_Function ) + { + // Execute job + (job.m_Function)( job.m_UserData ); + + // Immediately get another job. Ensure all jobs are consumed + // before shutting down. + continue; + } + + // Shutting down? + if ( m_WantToQuit.Load() ) + { + break; + } + + // Wait for work to be available + m_WakeSemaphore.Wait(); + } +} + +//------------------------------------------------------------------------------ +ThreadPool::ThreadPoolThread::ThreadPoolThread( uint32_t threadId, + ThreadPool & ownerPool ) + : m_ThreadId( threadId ) + , m_OwnerPool( ownerPool ) +{ + // Start thread + AStackString<> threadName; + threadName.Format( "ThreadPool_%02u", threadId ); + m_Thread.Start( ThreadFuncWrapper, threadName.Get(), this ); +} + +//------------------------------------------------------------------------------ +ThreadPool::ThreadPoolThread::~ThreadPoolThread() +{ + // Wait for thread to exit + m_Thread.Join(); +} + +//------------------------------------------------------------------------------ +void ThreadPool::ThreadPoolThread::ThreadFunc() +{ + PROFILE_FUNCTION; + + // First thread creates other threads + if ( ( m_ThreadId == 1 ) && ( m_OwnerPool.GetNumThreads() > 1 ) ) + { + PROFILE_SECTION("CreateAdditionalThreads"); + ThreadPoolThread * thread = this; + for ( uint32_t i = 2; i <= m_OwnerPool.GetNumThreads(); ++i ) + { + thread->m_Next = FNEW( ThreadPoolThread( i, m_OwnerPool ) ); + thread = thread->m_Next; + } + } + + // Process jobs + m_OwnerPool.ThreadFunc(); + + // First thread joins additional threads + if ( ( m_ThreadId == 1 ) && ( m_OwnerPool.GetNumThreads() > 1 ) ) + { + PROFILE_SECTION("DestroyAdditionalThreads"); + + // Join + ThreadPoolThread * thread = m_Next; + while ( thread ) + { + ThreadPoolThread * next = thread->m_Next; + FDELETE thread; + thread = next; + } + } +} + +//------------------------------------------------------------------------------ diff --git a/Code/Core/Process/ThreadPool.h b/Code/Core/Process/ThreadPool.h new file mode 100644 index 000000000..77989508a --- /dev/null +++ b/Code/Core/Process/ThreadPool.h @@ -0,0 +1,70 @@ +// ThreadPool +//------------------------------------------------------------------------------ +#pragma once + +// Includes +//------------------------------------------------------------------------------ +// Core +#include "Core/Containers/Array.h" +#include "Core/Containers/UniquePtr.h" +#include "Core/Process/Atomic.h" +#include "Core/Process/Mutex.h" +#include "Core/Process/Semaphore.h" +#include "Core/Process/Thread.h" + +// ThreadPool +//------------------------------------------------------------------------------ +class ThreadPool +{ +public: + explicit ThreadPool( uint32_t numThreads ); + ~ThreadPool(); + + // Enqueue jobs + using ThreadJobFunc = void(*)( void * param ); + void EnqueueJob( ThreadJobFunc func, void * userData = nullptr ); + + uint32_t GetNumThreads() const { return m_NumThreads; } + +protected: + void operator = (const ThreadPool & other) = delete; + + static uint32_t ThreadFuncWrapper( void * userData ); + void ThreadFunc(); + + // Job tracking structure + class ThreadJob + { + public: + ThreadJobFunc m_Function = nullptr; + void * m_UserData = nullptr; + }; + + // Thread structure + class ThreadPoolThread + { + public: + ThreadPoolThread( uint32_t threadId, ThreadPool & ownerPool ); + ~ThreadPoolThread(); + void ThreadFunc(); + + void operator = (const ThreadPoolThread & other) = delete; + protected: + uint32_t m_ThreadId = 0; + Thread m_Thread; + ThreadPoolThread * m_Next = nullptr; + ThreadPool & m_OwnerPool; + }; + + // Thread management + const uint32_t m_NumThreads = 0; + Atomic m_WantToQuit; + Semaphore m_WakeSemaphore; + ThreadPoolThread * m_FirstThread; + + // Job queue + Mutex m_JobQueueMutex; + Array< ThreadJob > m_JobQueue; +}; + +//------------------------------------------------------------------------------ From 958855dd5485a683c99a9b3cddcc616bb2ebdec8 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Jul 2023 15:44:50 +0930 Subject: [PATCH 059/129] GCC: Add config to build FASTBuild with gcc11 on Linux --- External/SDK/GCC/GCC.bff | 5 ++- External/SDK/GCC/Linux/GCC11.bff | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 External/SDK/GCC/Linux/GCC11.bff diff --git a/External/SDK/GCC/GCC.bff b/External/SDK/GCC/GCC.bff index 0d8b6aebc..36e4e2644 100644 --- a/External/SDK/GCC/GCC.bff +++ b/External/SDK/GCC/GCC.bff @@ -7,6 +7,7 @@ //#define USING_GCC_4 //#define USING_GCC_7 #define USING_GCC_9 + //#define USING_GCC_11 #endif // Activate @@ -21,8 +22,8 @@ #if USING_GCC_7 #include "Linux/GCC7.bff" #endif - #if USING_GCC_9 - #include "Linux/GCC9.bff" + #if USING_GCC_11 + #include "Linux/GCC11.bff" #endif #endif diff --git a/External/SDK/GCC/Linux/GCC11.bff b/External/SDK/GCC/Linux/GCC11.bff new file mode 100644 index 000000000..f132d3e55 --- /dev/null +++ b/External/SDK/GCC/Linux/GCC11.bff @@ -0,0 +1,66 @@ +// GCC 11.3.0 +//------------------------------------------------------------------------------ +.GCC11_BasePath = '/usr/bin' + +// Compiler +//------------------------------------------------------------------------------ +Compiler( 'Compiler-GCC11' ) +{ + .Executable = '$GCC11_BasePath$/x86_64-linux-gnu-gcc-11' + .ExtraFiles = { + '/usr/bin/as' + '/usr/lib/gcc/x86_64-linux-gnu/11/cc1' + '/usr/lib/gcc/x86_64-linux-gnu/11/cc1plus' + } + .CompilerFamily = 'gcc' // TODO: Remove when FASTBuild detection is improved + + // Allow tests to activate some experimental behavior + #if ENABLE_SOURCE_MAPPING + .SourceMapping_Experimental = '/fastbuild-test-mapping' + #endif +} + +// ToolChain +//------------------------------------------------------------------------------ +.ToolChain_GCC_Linux = +[ + .Platform = 'x64Linux' + + // Compiler Options + .Compiler = 'Compiler-GCC11' + .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output + + ' -c' // Compile only + + ' -g' // Generate debug info + + ' -m64' // x86_64 + + ' -D__LINUX__' // Platform define + + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h + + // Include paths + + ' -I./' + + // Enable warnings + + ' -Wall -Werror -Wfatal-errors' // warnings as errors + + ' -Wextra' + + ' -Wshadow' + + .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' + + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types + .CompilerOptionsC = ' -x c $CommonCompilerOptions$' + + // Librarian + .Librarian = '$GCC11_BasePath$/x86_64-linux-gnu-ar' + .LibrarianOptions = 'rcs "%2" "%1"' + + // Linker + .Linker = '$GCC11_BasePath$/x86_64-linux-gnu-g++-11' + .LinkerOptions = '"%1" -o "%2"' + + // File Extensions + .LibExtension = '.a' + .ExeExtension = '' + + // Exception Control + .UseExceptions = ' -fexceptions' +] + +//------------------------------------------------------------------------------ From 9b4af1633015ab43397b608aefb80b61cc8a9e77 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Jul 2023 15:45:09 +0930 Subject: [PATCH 060/129] Clang: Add config to build FASTBuild with Clang14 on Linux --- External/SDK/Clang/Clang.bff | 4 ++ External/SDK/Clang/Linux/Clang14.bff | 63 ++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 External/SDK/Clang/Linux/Clang14.bff diff --git a/External/SDK/Clang/Clang.bff b/External/SDK/Clang/Clang.bff index 37012d415..86f15a586 100644 --- a/External/SDK/Clang/Clang.bff +++ b/External/SDK/Clang/Clang.bff @@ -7,6 +7,7 @@ //#define USING_CLANG_3 //#define USING_CLANG_6 #define USING_CLANG_10 + //#define USING_CLANG_14 #endif #if __OSX__ && !CI_BUILD //#define USING_CLANG_8 @@ -36,6 +37,9 @@ #if USING_CLANG_10 #include "Linux/Clang10.bff" #endif + #if USING_CLANG_14 + #include "Linux/Clang14.bff" + #endif #endif #if __OSX__ #if CI_BUILD diff --git a/External/SDK/Clang/Linux/Clang14.bff b/External/SDK/Clang/Linux/Clang14.bff new file mode 100644 index 000000000..81ef2e505 --- /dev/null +++ b/External/SDK/Clang/Linux/Clang14.bff @@ -0,0 +1,63 @@ +// Clang 14.0.0 +//------------------------------------------------------------------------------ +.Clang14_BasePath = '/usr/lib/llvm-14/bin' + +// Compiler +//------------------------------------------------------------------------------ +Compiler( 'Compiler-Clang14' ) +{ + .Root = '$Clang14_BasePath$' + .Executable = '$Root$/clang' + + // Allow tests to activate some experimental behavior + #if ENABLE_RELATIVE_PATHS + .UseRelativePaths_Experimental = true + #endif + #if ENABLE_SOURCE_MAPPING + .SourceMapping_Experimental = '/fastbuild-test-mapping' + #endif +} + +// ToolChain +//------------------------------------------------------------------------------ +.ToolChain_Clang_Linux = +[ + .Platform = 'x64ClangLinux' + + // Compiler Options + .Compiler = 'Compiler-Clang14' + .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output + + ' -c' // Compile only + + ' -g' // Generate debug info + + ' -m64' // x86_64 + + ' -D__LINUX__' // Platform define + + // Include paths + + ' -I./' + + // Enable warnings + + ' -Wall -Werror -Wfatal-errors' // warnings as errors + + ' -Wextra' + + ' -Wshadow' + + .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' + + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types + .CompilerOptionsC = ' -x c $CommonCompilerOptions$' + + // Librarian + .Librarian = '$Clang14_BasePath$/llvm-ar' + .LibrarianOptions = 'rcs "%2" "%1"' + + // Linker + .Linker = '$Clang14_BasePath$/clang++' + .LinkerOptions = '"%1" -o "%2"' + + // File Extensions + .LibExtension = '.a' + .ExeExtension = '' + + // Exception Control + .UseExceptions = ' -fexceptions' +] + +//------------------------------------------------------------------------------ From c29648a6b28f5f88e2c4ea4792223585eb7a88f1 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 15 Jul 2023 15:47:33 +0930 Subject: [PATCH 061/129] GCC: Restore accidentally deleted Clang9 config for Linux --- External/SDK/GCC/GCC.bff | 3 +++ 1 file changed, 3 insertions(+) diff --git a/External/SDK/GCC/GCC.bff b/External/SDK/GCC/GCC.bff index 36e4e2644..67c7eeebd 100644 --- a/External/SDK/GCC/GCC.bff +++ b/External/SDK/GCC/GCC.bff @@ -22,6 +22,9 @@ #if USING_GCC_7 #include "Linux/GCC7.bff" #endif + #if USING_GCC_9 + #include "Linux/GCC9.bff" + #endif #if USING_GCC_11 #include "Linux/GCC11.bff" #endif From 396c5ec1df07561db9af107bc4462ee051c1133f Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 16 Jul 2023 10:10:21 +0930 Subject: [PATCH 062/129] Zstd: Fix compilation for Linux - disable use of asm - it's not used on other platforms: it would be worth investigating if this is a signifcant performance gain for our use -case or not --- External/Zstd/Zstd.bff | 127 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 External/Zstd/Zstd.bff diff --git a/External/Zstd/Zstd.bff b/External/Zstd/Zstd.bff new file mode 100644 index 000000000..f3547be78 --- /dev/null +++ b/External/Zstd/Zstd.bff @@ -0,0 +1,127 @@ +// Zstd +//------------------------------------------------------------------------------ +.ZstdBasePath = '../External/Zstd/zstd-1.5.5/lib' +.ZstdIncludePaths = ' "-I$ZstdBasePath$"' +{ + .ProjectName = 'Zstd' + .ProjectPath = '$ZstdBasePath$' + + // Target/Compiler specific options + .ZstdOptions_x64 = [ + .ZstdCompilerOptions = ' /wd4464' // relative include path contains '..' + + ' /wd4574' // '__has_attribute' is defined to be '0': did you mean to use '#if __has_attribute'? + + ' /wd6011' // Dereferencing NULL pointer 'ptr'. + + ' /wd6239' // ( && ) always evaluates to the result of . Did you intend to use the bitwise-and operator? + + ' /wd6293' // Ill-defined for-loop: counts down from minimum. + + ' /wd6326' // Potential comparison of a constant with another constant. + + ' /wd28251' // warning C28251: Inconsistent annotation for '_setjmp': this instance has no annotations. See (0). + + + ' -O2' // Compile with optimizations even in debug to improve performance + ] + .ZstdOptions_x64Clang = [ + .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + .ZstdOptions_x64Linux = [ + .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + + // Disable inline asm - TODO:B Is this a significant perf gain for Linux vs other platforms + // that don't seem to use it? + + ' -DZSTD_DISABLE_ASM' + ] + .ZstdOptions_x64ClangLinux = .ZstdOptions_x64Linux + .ZstdOptions_x64OSX = [ + .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + .ZstdOptions_ARMOSX = [ + .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + + // Library + //-------------------------------------------------------------------------- + .ProjectConfigs = {} + ForEach( .BuildConfig in .BuildConfigs ) + { + Using( .BuildConfig ) + + .OutputBase + '\$Platform$-$BuildConfigName$' + + Using( ."ZstdOptions_$Platform$" ) + + // Static Library + ObjectList( '$ProjectName$-Lib-$Platform$-$BuildConfigName$' ) + { + // Input - Only build specific files we use + .CompilerInputPattern = '*.c' + .CompilerInputPath = { + '$ZstdBasePath$/common/' + '$ZstdBasePath$/compress/' + '$ZstdBasePath$/decompress/' + } + + // Options + .CompilerOptions = .CompilerOptionsC + + .ZstdIncludePaths + + .ZstdCompilerOptions + + #if __WINDOWS__ + // Remove flags that disable opimizations + - ' /Od' + - ' /RTC1' + + // Disable clang-cl static analysis if enabled for this config + // (we won't fix warnings in 3rd party code) + - ' --analyze' + #else + - ' -O0' + #endif + + // Disable warnings if using Clang. There are too many warnings in Zstd + // and they differ with every version of Clang + - ' -Wall' + - ' -Werror' + - ' -Wfatal-errors' + - ' -Wextra' + - ' -Wshadow' + - ' -Weverything' + + // Output + .CompilerOutputPath = '$OutputBase$/External/$ProjectName$/' + } + Alias( '$ProjectName$-$Platform$-$BuildConfigName$' ) { .Targets = '$ProjectName$-Lib-$Platform$-$BuildConfigName$' } + + #if __WINDOWS__ + .ProjectConfig = [ Using( .'Project_$Platform$_$BuildConfigName$' ) .Target = '$ProjectName$-$Platform$-$BuildConfigName$' ] + ^ProjectConfigs + .ProjectConfig + #endif + #if __OSX__ + .ProjectConfig = [ .Config = '$BuildConfigName$' .Target = '$ProjectName$-x64OSX-$BuildConfigName$' ] + ^ProjectConfigs + .ProjectConfig + #endif + } + + // Aliases + //-------------------------------------------------------------------------- + CreateCommonAliases( .ProjectName ) + + // Visual Studio Project Generation + //-------------------------------------------------------------------------- + #if __WINDOWS__ + .ExtraOptions = [ + .ProjectFiles = '../External/Zstd/Zstd.bff' + ] + CreateVCXProject_Lib( .ProjectName, .ProjectPath, .ProjectConfigs, .ExtraOptions ) + #endif + + // XCode Project Generation + //-------------------------------------------------------------------------- + #if __OSX__ + XCodeProject( '$ProjectName$-xcodeproj' ) + { + .ProjectOutput = '../tmp/XCode/Projects/0_External/$ProjectName$.xcodeproj/project.pbxproj' + .ProjectInputPaths = '$ProjectPath$/' + .ProjectBasePath = '$ProjectPath$/' + + .XCodeBuildWorkingDir = '../../../../Code/' + } + #endif +} From 3343029ac804d48012c2d78f4a5f1e156a524955 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 16 Jul 2023 10:54:26 +0930 Subject: [PATCH 063/129] Zstd: Improve compile time (110 CPU seconds -> 50 CPU seconds) - exclude some files we don't need - use Unity to reduce redundant processing --- External/Zstd/Zstd.bff | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/External/Zstd/Zstd.bff b/External/Zstd/Zstd.bff index f3547be78..da3a7aae7 100644 --- a/External/Zstd/Zstd.bff +++ b/External/Zstd/Zstd.bff @@ -8,7 +8,8 @@ // Target/Compiler specific options .ZstdOptions_x64 = [ - .ZstdCompilerOptions = ' /wd4464' // relative include path contains '..' + .ZstdCompilerOptions = ' /wd4365' // conversion from '%s' to '%s', signed/unsigned mismatch + + ' /wd4464' // relative include path contains '..' + ' /wd4574' // '__has_attribute' is defined to be '0': did you mean to use '#if __has_attribute'? + ' /wd6011' // Dereferencing NULL pointer 'ptr'. + ' /wd6239' // ( && ) always evaluates to the result of . Did you intend to use the bitwise-and operator? @@ -47,16 +48,27 @@ Using( ."ZstdOptions_$Platform$" ) - // Static Library - ObjectList( '$ProjectName$-Lib-$Platform$-$BuildConfigName$' ) + Unity( '$ProjectName$-Unity-$Platform$-$BuildConfigName$' ) { - // Input - Only build specific files we use - .CompilerInputPattern = '*.c' - .CompilerInputPath = { + .UnityInputPattern = '*.c' + .UnityInputPath = { '$ZstdBasePath$/common/' '$ZstdBasePath$/compress/' '$ZstdBasePath$/decompress/' } + .UnityInputExcludedFiles = { + '$ZstdBasePath$/common/debug.c' + '$ZstdBasePath$/common/threading.c' + '$ZstdBasePath$/compress/zstdmt_compress.c' + } + .UnityNumFiles = 3 + .UnityOutputPath = '$OutputBase$/External/$ProjectName$/' + } + + // Static Library + ObjectList( '$ProjectName$-Lib-$Platform$-$BuildConfigName$' ) + { + .CompilerInputUnity = '$ProjectName$-Unity-$Platform$-$BuildConfigName$' // Options .CompilerOptions = .CompilerOptionsC From 27d7acc127060ccd57aef4a85d7d844a1f9f935e Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 16 Jul 2023 10:55:16 +0930 Subject: [PATCH 064/129] Zstd compression support groundwork: - extend internal helper and tests to support Zstd compression as well as LZ4 --- Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff | 1 + Code/Tools/FBuild/FBuild/FBuild.bff | 1 + Code/Tools/FBuild/FBuildCore/FBuildCore.bff | 1 + .../FBuild/FBuildCore/Helpers/Compressor.cpp | 105 ++++++++++++++++++ .../FBuild/FBuildCore/Helpers/Compressor.h | 4 + Code/Tools/FBuild/FBuildTest/FBuildTest.bff | 1 + .../FBuildTest/Tests/TestCompressor.cpp | 75 ++++++++++++- .../FBuild/FBuildWorker/FBuildWorker.bff | 1 + Code/fbuild.bff | 6 +- 9 files changed, 189 insertions(+), 6 deletions(-) diff --git a/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff b/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff index b44292908..21c722148 100644 --- a/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff +++ b/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff @@ -53,6 +53,7 @@ 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' 'xxHash-Lib-$Platform$-$BuildConfigName$' + 'Zstd-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/bfffuzzer$ExeExtension$' .LinkerOptions + ' -pthread -ldl -lrt' diff --git a/Code/Tools/FBuild/FBuild/FBuild.bff b/Code/Tools/FBuild/FBuild/FBuild.bff index af652c17e..79f93147b 100644 --- a/Code/Tools/FBuild/FBuild/FBuild.bff +++ b/Code/Tools/FBuild/FBuild/FBuild.bff @@ -50,6 +50,7 @@ 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' 'xxHash-Lib-$Platform$-$BuildConfigName$' + 'Zstd-Lib-$Platform$-$BuildConfigName$' } #if __LINUX__ .LinkerOutput = '$OutputBase$/$ProjectPath$/fbuild$ExeExtension$' // NOTE: lower case diff --git a/Code/Tools/FBuild/FBuildCore/FBuildCore.bff b/Code/Tools/FBuild/FBuildCore/FBuildCore.bff index 54232787c..a5d945afd 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildCore.bff +++ b/Code/Tools/FBuild/FBuildCore/FBuildCore.bff @@ -31,6 +31,7 @@ // Extra Compiler Options .CompilerOptions + .LZ4IncludePaths + + .ZstdIncludePaths .CompilerOptions + ' "-I../External/VSProjTypeExtractor"' // Output diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.cpp index 73b1c541c..404bff449 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.cpp @@ -19,6 +19,7 @@ // External #include "lz4.h" #include "lz4hc.h" +#include "zstd.h" #include @@ -172,4 +173,108 @@ bool Compressor::Decompress( const void * data ) return false; } +// CompressZstd +//------------------------------------------------------------------------------ +bool Compressor::CompressZstd( const void * data, + size_t dataSize, + int32_t compressionLevel) +{ + PROFILE_FUNCTION; + + ASSERT( data ); + ASSERT( m_Result == nullptr ); + + // allocate worst case output size for LZ4 + const size_t worstCaseSize = ZSTD_compressBound( dataSize ); + UniquePtr output( (char *)ALLOC( worstCaseSize ) ); + + size_t compressedSize; + + // do compression + if ( compressionLevel > 0 ) + { + compressedSize = ZSTD_compress( output.Get(), + worstCaseSize, + static_cast( data ), + dataSize, + compressionLevel); + } + else + { + // Disable compression + compressedSize = dataSize; // Act as if compression achieved nothing + } + + // did the compression yield any benefit? + const bool compressed = ( compressedSize < dataSize ); + + if (compressed) + { + // trim memory usage to compressed size + m_Result = ALLOC( (uint32_t)compressedSize + sizeof(Header) ); + memcpy( (char *)m_Result + sizeof(Header), output.Get(), (size_t)compressedSize ); + m_ResultSize = (uint32_t)compressedSize + sizeof(Header); + } + else + { + // compression failed, so just copy the old data + m_Result = ALLOC( dataSize + sizeof(Header) ); + memcpy( (char *)m_Result + sizeof(Header), data, dataSize ); + m_ResultSize = dataSize + sizeof(Header); + } + + // fill out header + Header * header = (Header *)m_Result; + header->m_CompressionType = compressed ? 2u : 0u; // compression type + header->m_UncompressedSize = (uint32_t)dataSize; // input size + header->m_CompressedSize = compressed ? (uint32_t)compressedSize : (uint32_t)dataSize; // output size + + return compressed; +} + +// DecompressZstd +//------------------------------------------------------------------------------ +bool Compressor::DecompressZstd( const void * data ) +{ + PROFILE_FUNCTION; + + ASSERT( data ); + ASSERT( m_Result == nullptr ); + + const Header * header = static_cast(data); + + // handle uncompressed case + if (header->m_CompressionType == 0) + { + m_Result = ALLOC(header->m_UncompressedSize); + memcpy(m_Result, (const char *)data + sizeof(Header), header->m_UncompressedSize); + m_ResultSize = header->m_UncompressedSize; + return true; + } + ASSERT( header->m_CompressionType == 2 ); + + // uncompressed size + const uint32_t uncompressedSize = header->m_UncompressedSize; + m_Result = ALLOC( uncompressedSize ); + m_ResultSize = uncompressedSize; + + // skip over header to LZ4 data + const char * compressedData = ( (const char *)data + sizeof(Header) ); + + // decompress + const size_t bytesDecompressed = ZSTD_decompress( m_Result, + uncompressedSize, + compressedData, + header->m_CompressedSize ); + if ( bytesDecompressed == uncompressedSize ) + { + return true; + } + + // Data is corrupt + FREE( m_Result ); + m_Result = nullptr; + m_ResultSize = 0; + return false; +} //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.h b/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.h index 5c8bd68b4..7f4c8e4ed 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.h +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.h @@ -24,6 +24,10 @@ class Compressor bool Compress( const void * data, size_t dataSize, int32_t compressionLevel = -1 ); // -1 = default LZ4 compression level bool Decompress( const void * data ); + // Zstd + bool CompressZstd( const void * data, size_t dataSize, int32_t compressionLevel = -1 ); // -1 = default Zstd compression level + bool DecompressZstd( const void * data ); + const void * GetResult() const { return m_Result; } size_t GetResultSize() const { return m_ResultSize; } diff --git a/Code/Tools/FBuild/FBuildTest/FBuildTest.bff b/Code/Tools/FBuild/FBuildTest/FBuildTest.bff index 509e41c1a..d5f694015 100644 --- a/Code/Tools/FBuild/FBuildTest/FBuildTest.bff +++ b/Code/Tools/FBuild/FBuildTest/FBuildTest.bff @@ -55,6 +55,7 @@ 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' 'xxHash-Lib-$Platform$-$BuildConfigName$' + 'Zstd-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/$ProjectName$$ExeExtension$' #if __WINDOWS__ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp index 941b6b703..77cb23277 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp @@ -31,7 +31,8 @@ class TestCompressor : public FBuildTest void CompressSimpleHelper( const char * data, size_t size, size_t expectedCompressedSize, - bool shouldCompress ) const; + bool shouldCompress, + bool useZstd = false ) const; void CompressHelper( const char * fileName ) const; }; @@ -76,7 +77,8 @@ void TestCompressor::CompressSimple() const void TestCompressor::CompressSimpleHelper( const char * data, size_t size, size_t expectedCompressedSize, - bool shouldCompress ) const + bool shouldCompress, + bool useZstd ) const { // raw input strings may not be aligned on Linux/OSX, so copy // them to achieve our required alignment @@ -86,7 +88,8 @@ void TestCompressor::CompressSimpleHelper( const char * data, // compress Compressor c; - const bool compressed = c.Compress( data, size ); + const bool compressed = useZstd ? c.CompressZstd( data, size ) + : c.Compress( data, size ); TEST_ASSERT( compressed == shouldCompress ); const size_t compressedSize = c.GetResultSize(); if ( expectedCompressedSize > 0 ) @@ -97,7 +100,14 @@ void TestCompressor::CompressSimpleHelper( const char * data, // decompress Compressor d; - TEST_ASSERT( d.Decompress( compressedMem ) ); + if ( useZstd ) + { + TEST_ASSERT( d.DecompressZstd( compressedMem ) ); + } + else + { + TEST_ASSERT( d.Decompress( compressedMem ) ); + } const size_t decompressedSize = d.GetResultSize(); TEST_ASSERT( decompressedSize == size ); TEST_ASSERT( memcmp( data, d.GetResult(), size ) == 0 ); @@ -140,6 +150,8 @@ void TestCompressor::CompressHelper( const char * fileName ) const OUTPUT( "Level | Time (ms) MB/s Ratio | Time (ms) MB/s\n" ); OUTPUT( "------------------------------------------------\n" ); + OUTPUT( "LZ4:\n" ); + // Compress at various compression levels const int32_t compressionLevels[] = { @@ -193,6 +205,61 @@ void TestCompressor::CompressHelper( const char * fileName ) const ( compressTimeTaken / numRepeats ), compressThroughputMBs, (double)ratio, ( decompressTimeTaken / numRepeats ), decompressThroughputMBs ); } + + OUTPUT( "Zstd:\n" ); + + // Compress at various compression levels + const int32_t zStdCompressionLevels[] = + { + 0, // Disabled + 1, 3, 6, 9, 12, 15, 18, 21 // Zstd + }; + + for ( const int32_t compressionLevel : zStdCompressionLevels ) + { + // compress/decompress the data several times to get more stable throughput value + const uint32_t numRepeats = 4; // Increase to get more consistent numbers + double compressTimeTaken = 0.0; + double decompressTimeTaken = 0.0; + uint64_t compressedSize = 0; + + // Compression speed + UniquePtr< Compressor, DeleteDeletor > c; + for ( uint32_t i = 0; i < numRepeats; ++i ) + { + // Compress + c = FNEW( Compressor ); + const Timer t; + c.Get()->CompressZstd( data.Get(), dataSize, compressionLevel ); + compressedSize = c.Get()->GetResultSize(); + compressTimeTaken += (double)t.GetElapsedMS(); + } + + // Decompression speed + for ( uint32_t i = 0; i < numRepeats; ++i ) + { + // Decompress + const Timer t2; + Compressor d; + TEST_ASSERT( d.DecompressZstd( c.Get()->GetResult() ) ); + TEST_ASSERT( d.GetResultSize() == dataSize ); + decompressTimeTaken += (double)t2.GetElapsedMS(); + + // Sanity check decompression returns original results + if ( i == 0 ) + { + TEST_ASSERT( memcmp( data.Get(), d.GetResult(), dataSize ) == 0 ); + } + } + + const double compressThroughputMBs = ( ( (double)dataSize * (double)numRepeats ) / ( compressTimeTaken / 1000.0 ) ) / (double)MEGABYTE; + const double decompressThroughputMBs = ( ( (double)dataSize * (double)numRepeats ) / ( decompressTimeTaken / 1000.0 ) ) / (double)MEGABYTE; + const double ratio = ( (double)dataSize / (double)compressedSize ); + + OUTPUT( "%-5i | %8.3f %7.1f %5.2f | %8.3f %7.1f\n", compressionLevel, + ( compressTimeTaken / numRepeats ), compressThroughputMBs, (double)ratio, + ( decompressTimeTaken / numRepeats ), decompressThroughputMBs ); + } OUTPUT( "------------------------------------------------\n" ); } diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff index 461c31fd4..cf03fc6e9 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff @@ -80,6 +80,7 @@ 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' 'xxHash-Lib-$Platform$-$BuildConfigName$' + 'Zstd-Lib-$Platform$-$BuildConfigName$' } #if __WINDOWS__ + 'FBuildWorker-Res-$Platform$-$BuildConfigName$' diff --git a/Code/fbuild.bff b/Code/fbuild.bff index 9b6e2bb4a..cbeb7b1c7 100644 --- a/Code/fbuild.bff +++ b/Code/fbuild.bff @@ -400,6 +400,7 @@ Settings // External #include "..\External\LZ4\LZ4.bff" #include "../External/xxHash/xxHash.bff" +#include "../External/Zstd/Zstd.bff" // Test Framework #include "TestFramework\TestFramework.bff" @@ -657,7 +658,7 @@ Alias( 'All+Tests' ) .Folder_External = [ .Path = 'External' - .Projects = { 'LZ4-proj', 'SDKs-proj', 'xxHash-proj' } + .Projects = { 'LZ4-proj', 'SDKs-proj', 'xxHash-proj', 'Zstd-proj' } ] .Folder_Test = [ @@ -698,7 +699,8 @@ Alias( 'All+Tests' ) 'LZ4-xcodeproj' 'OSUI-xcodeproj' 'TestFramework-xcodeproj' - 'xxHash-xcodeproj' } + 'xxHash-xcodeproj' + 'Zstd-xcodeproj' } .ProjectConfigs = {} ForEach( .BuildConfig in .BuildConfigs ) { From 21bd4f6665dea4f26c74f9533805e37bb48bbd70 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 16 Jul 2023 11:16:01 +0930 Subject: [PATCH 065/129] Ztsd: Fix Zstd compilation error - Fix Windows static analysis config - disable asm dependencies on x86_64 OSX like Linux (as with Linux, this needs investigation to understand the performance impact) --- External/Zstd/Zstd.bff | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/External/Zstd/Zstd.bff b/External/Zstd/Zstd.bff index da3a7aae7..6cc807851 100644 --- a/External/Zstd/Zstd.bff +++ b/External/Zstd/Zstd.bff @@ -15,7 +15,11 @@ + ' /wd6239' // ( && ) always evaluates to the result of . Did you intend to use the bitwise-and operator? + ' /wd6293' // Ill-defined for-loop: counts down from minimum. + ' /wd6326' // Potential comparison of a constant with another constant. - + ' /wd28251' // warning C28251: Inconsistent annotation for '_setjmp': this instance has no annotations. See (0). + + ' /wd26448' // Consider using gsl::finally if final action is intended (gsl.util). + + ' /wd26462' // The value pointed to by '%s' is assigned only once, mark it as a pointer to const + + ' /wd26818' // Switch statement does not cover all cases. Consider adding a 'default' label (es.79). + + ' /wd26819' // Unannotated fallthrough between switch labels (es.78). + + ' /wd28251' // Inconsistent annotation for '_setjmp': this instance has no annotations. See (0). + ' -O2' // Compile with optimizations even in debug to improve performance ] @@ -32,6 +36,10 @@ .ZstdOptions_x64ClangLinux = .ZstdOptions_x64Linux .ZstdOptions_x64OSX = [ .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + + // Disable inline asm - TODO:B Is this a significant perf gain for Linux vs other platforms + // that don't seem to use it? + + ' -DZSTD_DISABLE_ASM' ] .ZstdOptions_ARMOSX = [ .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance From b63c606e92a0c8c017a8c55dfc0a5c8814e053bd Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 16 Jul 2023 11:45:31 +0930 Subject: [PATCH 066/129] JobQueue/JobQueueRemote use ThreadPool: - prepares for some future additional use of the ThreadPool --- .../FBuild/FBuildCore/WorkerPool/JobQueue.cpp | 25 +++++++++++++------ .../FBuild/FBuildCore/WorkerPool/JobQueue.h | 2 ++ .../FBuildCore/WorkerPool/JobQueueRemote.cpp | 9 ++++++- .../FBuildCore/WorkerPool/JobQueueRemote.h | 2 ++ .../FBuildCore/WorkerPool/WorkerThread.cpp | 17 +++---------- .../FBuildCore/WorkerPool/WorkerThread.h | 7 +++--- 6 files changed, 36 insertions(+), 26 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp index 381c200a5..def244ff0 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp @@ -16,7 +16,7 @@ #include "Core/Time/Timer.h" #include "Core/FileIO/FileIO.h" #include "Core/Process/Atomic.h" -#include "Core/Process/Thread.h" +#include "Core/Process/ThreadPool.h" #include "Core/Profile/Profile.h" // JobCostSorter @@ -161,14 +161,21 @@ JobQueue::JobQueue( uint32_t numWorkerThreads ) : WorkerThread::InitTmpDir(); - for ( uint32_t i=0; i 0 ) { - // identify each worker with an id starting from 1 - // (the "main" thread is considered 0) - const uint16_t threadIndex = static_cast( i + 1 ); - WorkerThread * wt = FNEW( WorkerThread( threadIndex ) ); - wt->Init(); - m_Workers.Append( wt ); + // Create thread pool + m_ThreadPool = FNEW( ThreadPool( numWorkerThreads ) ); + + // Create a job to run on each thread + for ( uint32_t i = 0; i < numWorkerThreads; ++i ) + { + // identify each worker with an id starting from 1 + // (the "main" thread is considered 0) + const uint16_t threadIndex = static_cast( i + 1 ); + WorkerThread * wt = FNEW( WorkerThread( threadIndex ) ); + wt->Init( m_ThreadPool ); + m_Workers.Append( wt ); + } } } @@ -211,6 +218,8 @@ JobQueue::~JobQueue() ASSERT( m_CompletedJobs.IsEmpty() ); ASSERT( m_CompletedJobsFailed.IsEmpty() ); ASSERT( Job::GetTotalLocalDataMemoryUsage() == 0 ); + + FDELETE m_ThreadPool; } // SignalStopWorkers (Main Thread) diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h index d8e96c8d5..e75cc9c3c 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h @@ -15,6 +15,7 @@ //------------------------------------------------------------------------------ class Node; class Job; +class ThreadPool; class WorkerThread; @@ -117,6 +118,7 @@ class JobQueue : public Singleton< JobQueue > Array< Job * > m_CompletedJobs2; Array< Job * > m_CompletedJobsFailed2; + ThreadPool * m_ThreadPool = nullptr; Array< WorkerThread * > m_Workers; }; diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp index 2774a4e90..bbba846c8 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp @@ -19,6 +19,7 @@ #include "Core/FileIO/FileIO.h" #include "Core/FileIO/FileStream.h" #include "Core/FileIO/PathUtils.h" +#include "Core/Process/ThreadPool.h" #include "Core/Profile/Profile.h" #include "Core/Time/Timer.h" #include "Core/Tracing/Tracing.h" @@ -33,13 +34,17 @@ JobQueueRemote::JobQueueRemote( uint32_t numWorkerThreads ) : { WorkerThread::InitTmpDir( true ); // remote == true + // Create thread pool + m_ThreadPool = FNEW( ThreadPool( numWorkerThreads ) ); + + // Create a job to run on each thread for ( uint32_t i=0; i( i + 1001 ); WorkerThread * wt = FNEW( WorkerThreadRemote( threadIndex ) ); - wt->Init(); + wt->Init( m_ThreadPool ); m_Workers.Append( wt ); } } @@ -58,6 +63,8 @@ JobQueueRemote::~JobQueueRemote() m_Workers[ i ]->WaitForStop(); FDELETE m_Workers[ i ]; } + + FDELETE m_ThreadPool; } // SignalStopWorkers (Main Thread) diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.h index 4ec44799f..955b8dd80 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.h @@ -15,6 +15,7 @@ //------------------------------------------------------------------------------ class Node; class Job; +class ThreadPool; class WorkerThread; // JobQueueRemote @@ -67,6 +68,7 @@ class JobQueueRemote : public Singleton< JobQueueRemote > Semaphore m_WorkerThreadSemaphore; Semaphore m_WorkerThreadSleepSemaphore; + ThreadPool * m_ThreadPool; Array< WorkerThread * > m_Workers; }; diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp index 95bea6cf3..88a6473d7 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp @@ -17,7 +17,7 @@ #include "Core/FileIO/FileStream.h" #include "Core/FileIO/PathUtils.h" #include "Core/Process/Atomic.h" -#include "Core/Process/Thread.h" +#include "Core/Process/ThreadPool.h" #include "Core/Profile/Profile.h" // Static @@ -36,19 +36,17 @@ WorkerThread::WorkerThread( uint16_t threadIndex ) // Init //------------------------------------------------------------------------------ -void WorkerThread::Init() +void WorkerThread::Init( ThreadPool * pool ) { PROFILE_FUNCTION; - // Start thread - m_Thread.Start( ThreadWrapperFunc, "WorkerThread", this, MEGABYTE ); + pool->EnqueueJob( ThreadWrapperFunc, this ); } //------------------------------------------------------------------------------ WorkerThread::~WorkerThread() { ASSERT( m_Exited.Load() ); - m_Thread.Join(); } // InitTmpDir @@ -107,19 +105,12 @@ void WorkerThread::WaitForStop() // MainWrapper //------------------------------------------------------------------------------ -/*static*/ uint32_t WorkerThread::ThreadWrapperFunc( void * param ) +/*static*/ void WorkerThread::ThreadWrapperFunc( void * param ) { WorkerThread * wt = static_cast< WorkerThread * >( param ); s_WorkerThreadThreadIndex = wt->m_ThreadIndex; - #if defined( PROFILING_ENABLED ) - AStackString<> threadName; - threadName.Format( "%s_%02u", s_WorkerThreadThreadIndex > 1000 ? "RemoteWorkerThread" : "WorkerThread", s_WorkerThreadThreadIndex ); - PROFILE_SET_THREAD_NAME( threadName.Get() ); - #endif - wt->Main(); - return 0; } // Main diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h index 61ef5b3ad..a146fa5b2 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h @@ -8,13 +8,13 @@ #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" #include "Core/Process/Semaphore.h" -#include "Core/Process/Thread.h" #include "Core/Strings/AStackString.h" #include "Core/Strings/AString.h" // Forward Declarations //------------------------------------------------------------------------------ class FileStream; +class ThreadPool; // WorkerThread //------------------------------------------------------------------------------ @@ -22,7 +22,7 @@ class WorkerThread { public: explicit WorkerThread( uint16_t threadIndex ); - void Init(); + void Init( ThreadPool * pool ); virtual ~WorkerThread(); static void InitTmpDir( bool remote = false ); @@ -46,11 +46,10 @@ class WorkerThread static bool Update(); // worker thread main loop - static uint32_t ThreadWrapperFunc( void * param ); + static void ThreadWrapperFunc( void * param ); virtual void Main(); // signal to exit thread - Thread m_Thread; Atomic m_ShouldExit; Atomic m_Exited; uint16_t m_ThreadIndex; From f5f5f464d2a3f694ab5a18cf1bae2fe6e3dac59d Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 16 Jul 2023 12:40:16 +0930 Subject: [PATCH 067/129] TestExec: Fix Exec test invoking Build twice on the same FBuild instance - this is not valid and is also inefficient --- Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp index 2d3ed395f..71ef3a07a 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp @@ -322,15 +322,14 @@ void TestExec::Build_ExecEnvCommand() const FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); - // build (via alias) - TEST_ASSERT( fBuild.Build( "EnvHelperExe" ) ); + // Build and run exe that checks env var is set + // (the executable checks the expected env var) + TEST_ASSERT( fBuild.Build( "ExecEnvCommandTest" ) ); // Check stats // Seen, Built, Type CheckStatsNode ( 1, 1, Node::EXE_NODE ); - - // Run the execenv command and ensure we get the expected output - TEST_ASSERT( fBuild.Build( "ExecEnvCommandTest" ) ); + CheckStatsNode ( 1, 1, Node::EXEC_NODE ); } // Exclusions From b2ef65e8117e139985133fe2258023d10f1ed73a Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 16 Jul 2023 13:14:46 +0930 Subject: [PATCH 068/129] SmallBlockAllocator: Remove single-threaded mode - while faster, this mode is unsafe and anything so allocation heavy as to be significantly impacted by this can probably be optimized more effectively in other ways --- .../Tests/TestSmallBlockAllocator.cpp | 20 ++----- Code/Core/Mem/SmallBlockAllocator.cpp | 60 ++----------------- Code/Core/Mem/SmallBlockAllocator.h | 13 ---- Code/Tools/FBuild/FBuildCore/FBuild.cpp | 4 -- 4 files changed, 8 insertions(+), 89 deletions(-) diff --git a/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp b/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp index 819af83fb..bb110bd94 100644 --- a/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp +++ b/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp @@ -39,7 +39,7 @@ class TestSmallBlockAllocator : public TestGroup // Helper functions static void GetRandomAllocSizes( const uint32_t numAllocs, Array< uint32_t> & allocSizes ); static float AllocateFromSystemAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount ); - static float AllocateFromSmallBlockAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount, const bool threadSafe = true ); + static float AllocateFromSmallBlockAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount ); static uint32_t ThreadFunction_System( void * userData ); static uint32_t ThreadFunction_SmallBlock( void * userData ); }; @@ -67,12 +67,10 @@ void TestSmallBlockAllocator::SingleThreaded() const const float time1 = AllocateFromSystemAllocator( allocSizes, repeatCount ); const float time2 = AllocateFromSmallBlockAllocator( allocSizes, repeatCount ); - const float time3 = AllocateFromSmallBlockAllocator( allocSizes, repeatCount, false ); // Thread-safe = false // output - OUTPUT( "System (malloc) : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time1, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time1 ) ); - OUTPUT( "SmallBlockAllocator : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time2, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time2 ) ); - OUTPUT( "SmallBlockAllocator (Single-Threaded mode) : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time3, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time3 ) ); + OUTPUT( "System (malloc) : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time1, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time1 ) ); + OUTPUT( "SmallBlockAllocator : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time2, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time2 ) ); } // MultiThreaded @@ -191,18 +189,13 @@ void TestSmallBlockAllocator::MultiThreaded() const // AllocateFromSmallBlockAllocator //------------------------------------------------------------------------------ -/*static*/ float TestSmallBlockAllocator::AllocateFromSmallBlockAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount, const bool threadSafe ) +/*static*/ float TestSmallBlockAllocator::AllocateFromSmallBlockAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount ) { const size_t numAllocs = allocSizes.GetSize(); Array< void * > allocs( numAllocs, false ); const Timer timer; - if ( threadSafe == false ) - { - SmallBlockAllocator::SetSingleThreadedMode( true ); - } - for ( size_t r = 0; r < repeatCount; ++r ) { // Use ALLOC @@ -222,11 +215,6 @@ void TestSmallBlockAllocator::MultiThreaded() const allocs.Clear(); } - if ( threadSafe == false ) - { - SmallBlockAllocator::SetSingleThreadedMode( false ); - } - return timer.GetElapsed(); } diff --git a/Code/Core/Mem/SmallBlockAllocator.cpp b/Code/Core/Mem/SmallBlockAllocator.cpp index d56fb85f7..d03502055 100644 --- a/Code/Core/Mem/SmallBlockAllocator.cpp +++ b/Code/Core/Mem/SmallBlockAllocator.cpp @@ -32,10 +32,6 @@ // Static Data //------------------------------------------------------------------------------ -/*static*/ bool SmallBlockAllocator::s_ThreadSafeAllocs( true ); -#if defined( ASSERTS_ENABLED ) - /*static*/ uint64_t SmallBlockAllocator::s_ThreadSafeAllocsDebugOwnerThread( 0 ); -#endif /*static*/ void * SmallBlockAllocator::s_BucketMemoryStart( MEM_BUCKETS_NOT_INITIALIZED ); /*static*/ uint32_t SmallBlockAllocator::s_BucketNextFreePageIndex( 0 ); /*static*/ uint64_t SmallBlockAllocator::s_BucketMemBucketMemory[ BUCKET_NUM_BUCKETS * sizeof( MemBucket ) / sizeof (uint64_t) ]; @@ -143,19 +139,11 @@ void * SmallBlockAllocator::Alloc( size_t size, size_t align ) return nullptr; // Can't satify alignment } - // Sanity check that we're being used safely - ASSERT( s_ThreadSafeAllocs || ( s_ThreadSafeAllocsDebugOwnerThread == (uint64_t)Thread::GetCurrentThreadId() ) ); - - void* ptr; + void* ptr; // Alloc - if ( s_ThreadSafeAllocs ) - { - MutexHolder mh(bucket.m_Mutex); - ptr = bucket.Alloc(); - } - else { + MutexHolder mh( bucket.m_Mutex ); ptr = bucket.Alloc(); } @@ -193,55 +181,15 @@ bool SmallBlockAllocator::Free( void * ptr ) MemDebug::FillMem( ptr, bucket.m_BlockSize, MemDebug::MEM_FILL_FREED_ALLOCATION_PATTERN ); #endif - // Sanity check that we're being used safely - ASSERT( s_ThreadSafeAllocs || ( s_ThreadSafeAllocsDebugOwnerThread == (uint64_t)Thread::GetCurrentThreadId() ) ); - // Free it - if ( s_ThreadSafeAllocs ) - { - bucket.m_Mutex.Lock(); - } - - bucket.Free( ptr ); - - if ( s_ThreadSafeAllocs ) { - bucket.m_Mutex.Unlock(); + MutexHolder mh( bucket.m_Mutex ); + bucket.Free( ptr ); } return true; } -// SetSingleThreadedMode -//------------------------------------------------------------------------------ -/*static*/ void SmallBlockAllocator::SetSingleThreadedMode( bool singleThreadedMode ) -{ - if ( singleThreadedMode ) - { - // Sanity check we're not already in single threaded mode - ASSERT( s_ThreadSafeAllocs == true ); - ASSERT( s_ThreadSafeAllocsDebugOwnerThread == 0 ); - - // Store the new owner thread for further safety checks - #if defined( ASSERTS_ENABLED ) - s_ThreadSafeAllocsDebugOwnerThread = (uint64_t)Thread::GetCurrentThreadId(); - #endif - } - else - { - // Sanity check we're in single threaded mode - ASSERT( s_ThreadSafeAllocs == false ); - ASSERT( s_ThreadSafeAllocsDebugOwnerThread == (uint64_t)Thread::GetCurrentThreadId() ); - - // Store the new owner thread for further safety checks - #if defined( ASSERTS_ENABLED ) - s_ThreadSafeAllocsDebugOwnerThread = 0; - #endif - } - - s_ThreadSafeAllocs = ( !singleThreadedMode ); -} - // AllocateMemoryForPage //------------------------------------------------------------------------------ /*virtual*/ void * SmallBlockAllocator::MemBucket::AllocateMemoryForPage() diff --git a/Code/Core/Mem/SmallBlockAllocator.h b/Code/Core/Mem/SmallBlockAllocator.h index 4364ef6f0..bcc49303f 100644 --- a/Code/Core/Mem/SmallBlockAllocator.h +++ b/Code/Core/Mem/SmallBlockAllocator.h @@ -19,7 +19,6 @@ // Includes //------------------------------------------------------------------------------ -#include "Core/Env/Assert.h" #include "Core/Mem/MemPoolBlock.h" #include "Core/Process/Mutex.h" @@ -34,9 +33,6 @@ // Attempt to free. Returns false if not a bucket owned allocation static bool Free( void * ptr ); - // Hint when operating only on a single thread as we can greatly reduce allocation cost - static void SetSingleThreadedMode( bool singleThreadedMode ); - #if defined( DEBUG ) static void DumpStats(); #endif @@ -69,15 +65,6 @@ Mutex m_Mutex; }; - friend class MemBucket; - - // Single Threaded Mode - static bool s_ThreadSafeAllocs; - #if defined( ASSERTS_ENABLED ) - // When in single-threaded mode, catch unsafe use - static uint64_t s_ThreadSafeAllocsDebugOwnerThread; - #endif - // Address space used by allocators static void * s_BucketMemoryStart; static uint32_t s_BucketNextFreePageIndex; // Next free memory page to commit diff --git a/Code/Tools/FBuild/FBuildCore/FBuild.cpp b/Code/Tools/FBuild/FBuildCore/FBuild.cpp index d602c435c..bae1ee809 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuild.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuild.cpp @@ -176,12 +176,8 @@ bool FBuild::Initialize( const char * nodeGraphDBFile ) } } - SmallBlockAllocator::SetSingleThreadedMode( true ); - m_DependencyGraph = NodeGraph::Initialize( bffFile, m_DependencyGraphFile.Get(), m_Options.m_ForceDBMigration_Debug ); - SmallBlockAllocator::SetSingleThreadedMode( false ); - if ( m_DependencyGraph == nullptr ) { return false; From 5a69424a1eded797d0c4d37ffa629cbb0daacd57 Mon Sep 17 00:00:00 2001 From: Domain Date: Sun, 6 Aug 2023 14:36:56 +0800 Subject: [PATCH 069/129] get limit from system configuration (#923) * get limit from system configuration * args not long enough --- Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp | 4 +++- Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp index 498c4f561..92e0f21d7 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp @@ -14,6 +14,8 @@ // system #if defined( __OSX__ ) #include +#elif defined( __LINUX__ ) + #include #endif // CONSTRUCTOR @@ -112,7 +114,7 @@ bool Args::Finalize( const AString & exe, const AString & nodeNameForError, Args const uint32_t argLimit( ARG_MAX - 1 ); #elif defined( __LINUX__ ) // On Linux it's problematic to reliably determine this, so we make a best guess - const uint32_t argLimit( ( 128 * 1024 ) - 1 ); + const uint32_t argLimit( (uint32_t)sysconf(_SC_ARG_MAX) ); #endif // If the args exceed the cmd line limit, a response file is required diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp index c17550bcc..489b145f6 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp @@ -144,8 +144,8 @@ void TestArgs::Check( ArgsResponseFileMode mode, if ( longArgs ) { - // Add ~3200 KiB of command line args ( 32 * 100 * 1024 ) - for ( size_t i = 0; i < 100 * 1024; ++i ) + // Add ~3200 KiB of command line args ( 32 * 1000 * 1024 ) + for ( size_t i = 0; i < 1000 * 1024; ++i ) { args += "123456789012345678901234567890X"; // 31 chars args.AddDelimiter(); From 374a44e9ae698a751c3ebeec54b99e9320ce50d9 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 6 Aug 2023 16:27:23 +0930 Subject: [PATCH 070/129] Post-integration cleanup: Linux args limit detection improvement - minor style fixes - improve/update comments - tweak args limit size for tests --- Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp | 6 ++++-- Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp index 92e0f21d7..37650d6e8 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp @@ -113,8 +113,10 @@ bool Args::Finalize( const AString & exe, const AString & nodeNameForError, Args #elif defined( __OSX__ ) const uint32_t argLimit( ARG_MAX - 1 ); #elif defined( __LINUX__ ) - // On Linux it's problematic to reliably determine this, so we make a best guess - const uint32_t argLimit( (uint32_t)sysconf(_SC_ARG_MAX) ); + // This doesn't account for some things that impact max arg length such + // as the size of the environment block, but it's better than using an + // arbitrary value + const uint32_t argLimit = static_cast( sysconf( _SC_ARG_MAX ) ); #endif // If the args exceed the cmd line limit, a response file is required diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp index 489b145f6..ef69b4b21 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp @@ -144,8 +144,8 @@ void TestArgs::Check( ArgsResponseFileMode mode, if ( longArgs ) { - // Add ~3200 KiB of command line args ( 32 * 1000 * 1024 ) - for ( size_t i = 0; i < 1000 * 1024; ++i ) + // Add ~32 MiB of command line args ( 32 * 1024 * 1024 ) + for ( size_t i = 0; i < 1024 * 1024; ++i ) { args += "123456789012345678901234567890X"; // 31 chars args.AddDelimiter(); From 91541cde8c82901b6fbbab87702aeb977abd8e0b Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 6 Aug 2023 17:54:37 +0930 Subject: [PATCH 071/129] [Fix][Linux] Robustify file copy if sendfile isn't supported (Thanks to Nick Krecklow) - if sendfile fails, copy contents of file manually in 4KiB chunks --- Code/Core/FileIO/FileIO.cpp | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Code/Core/FileIO/FileIO.cpp b/Code/Core/FileIO/FileIO.cpp index 97ee6fab8..ad29381d1 100644 --- a/Code/Core/FileIO/FileIO.cpp +++ b/Code/Core/FileIO/FileIO.cpp @@ -260,6 +260,8 @@ ssize_t bytesCopied = 0; ssize_t offset = 0; + bool sendfileUnavailable = false; + while ( offset < stat_source.st_size ) { // sendfile has an arbitrary limit of 0x7ffff000 on all systems (even if 64bit) @@ -269,11 +271,46 @@ const ssize_t sent = sendfile( dest, source, &offset, count ); if ( sent <= 0 ) { + // sendfile manual suggests defaulting to read/write functions + if ( ( sent == -1 ) && ( ( errno == EINVAL ) || ( errno == ENOSYS ) ) ) + { + sendfileUnavailable = true; + } + break; // Copy failed (incomplete) } bytesCopied += sent; } + // manually copy source file to destination in fixed-size chunks + // continues until source EOF is reached or any data fails to copy + if ( sendfileUnavailable ) + { + const size_t count = 4096; + void * buf = ALLOC( count ); + + while ( bytesCopied < stat_source.st_size ) + { + const ssize_t readBytes = read( source, buf, count ); + + if ( readBytes <= 0 ) + { + break; + } + + const ssize_t written = write( dest, buf, readBytes ); + + if ( written != readBytes ) + { + break; + } + + bytesCopied += written; + } + + FREE( buf ); + } + close( source ); close( dest ); From 54f4beab2b16b3998c7fdb695d90972bff121a9a Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 13 Aug 2023 12:06:59 +0930 Subject: [PATCH 072/129] VCProjectGenerator: Tidy up loops - use range-based for to simplify some loops --- .../FBuildCore/Helpers/VSProjectGenerator.cpp | 147 ++++++++---------- 1 file changed, 68 insertions(+), 79 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp index 2fa4ccd16..1ead2f2b2 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp @@ -73,10 +73,9 @@ void VSProjectGenerator::AddFile( const AString & file ) //------------------------------------------------------------------------------ void VSProjectGenerator::AddFiles( const Array< AString > & files ) { - const AString * const fEnd = files.End(); - for ( const AString * fIt = files.Begin(); fIt!=fEnd; ++fIt ) + for ( const AString & file : files ) { - AddFile( *fIt ); + AddFile( file ); } } @@ -115,12 +114,11 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile // Project Configurations { Write( " \n" ); - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF(" \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); - WriteF(" %s\n", cIt->m_Config.Get() ); - WriteF(" %s\n", cIt->m_Platform.Get() ); + WriteF(" \n", config.m_Config.Get(), config.m_Platform.Get() ); + WriteF(" %s\n", config.m_Config.Get() ); + WriteF(" %s\n", config.m_Platform.Get() ); Write( " \n" ); } Write( " \n" ); @@ -133,20 +131,19 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile { const AString & fileName = filePathPair.m_ProjectRelativePath; - const char * fileType = nullptr; - const VSProjectFileType * const end = fileTypes.End(); - for ( const VSProjectFileType * it=fileTypes.Begin(); it!=end; ++it ) + const char * fileTypeToUse = nullptr; + for ( const VSProjectFileType & fileType : fileTypes ) { - if ( AString::MatchI( it->m_Pattern.Get(), fileName.Get() ) ) + if ( AString::MatchI( fileType.m_Pattern.Get(), fileName.Get() ) ) { - fileType = it->m_FileType.Get(); + fileTypeToUse = fileType.m_FileType.Get(); break; } } - if ( fileType ) + if ( fileTypeToUse ) { WriteF( " \n", fileName.Get() ); - WriteF( " %s\n", fileType ); + WriteF( " %s\n", fileTypeToUse ); Write( " \n" ); } else @@ -162,10 +159,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile Write(" \n" ); { // Project References - const AString * const end = m_ProjectReferences.End(); - for ( const AString * it = m_ProjectReferences.Begin(); it != end; ++it ) + for ( const AString & projectReference : m_ProjectReferences ) { - AStackString<> proj( *it ); + AStackString<> proj( projectReference ); const char * pipe = proj.Find( '|' ); if ( pipe ) { @@ -183,10 +179,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile } { // References - const AString * const end = m_References.End(); - for ( const AString * it = m_References.Begin(); it != end; ++it ) + for ( const AString & reference : m_References ) { - WriteF( " \n", it->Get() ); + WriteF( " \n", reference.Get() ); } } Write(" \n" ); @@ -236,30 +231,29 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile // Configurations { - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF( " \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); + WriteF( " \n", config.m_Config.Get(), config.m_Platform.Get() ); Write( " Makefile\n" ); Write( " false\n" ); // If a specific executable is specified, use that, otherwise try to auto-derive // the executable from the .Target - AStackString<> localDebuggerCommand( cIt->m_LocalDebuggerCommand ); + AStackString<> localDebuggerCommand( config.m_LocalDebuggerCommand ); if ( localDebuggerCommand.IsEmpty() ) { // Get the executable path and make it project-relative - const Node * debugTarget = ProjectGeneratorBase::FindExecutableDebugTarget( cIt->m_TargetNode ); + const Node * debugTarget = ProjectGeneratorBase::FindExecutableDebugTarget( config.m_TargetNode ); if ( debugTarget ) { ProjectGeneratorBase::GetRelativePath( projectBasePath, debugTarget->GetName(), localDebuggerCommand ); } } - WritePGItem( "PlatformToolset", cIt->m_PlatformToolset ); - WritePGItem( "LocalDebuggerCommandArguments", cIt->m_LocalDebuggerCommandArguments ); + WritePGItem( "PlatformToolset", config.m_PlatformToolset ); + WritePGItem( "LocalDebuggerCommandArguments", config.m_LocalDebuggerCommandArguments ); WritePGItem( "LocalDebuggerCommand", localDebuggerCommand ); - WritePGItem( "LocalDebuggerEnvironment", cIt->m_LocalDebuggerEnvironment ); + WritePGItem( "LocalDebuggerEnvironment", config.m_LocalDebuggerEnvironment ); Write( " \n" ); } @@ -274,10 +268,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile // Property Sheets { - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF(" \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); + WriteF(" \n", config.m_Config.Get(), config.m_Platform.Get() ); Write( " \n" ); Write( " \n" ); } @@ -288,34 +281,33 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile // Property Group { - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF( " \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); + WriteF( " \n", config.m_Config.Get(), config.m_Platform.Get() ); - if ( cIt->m_Keyword == "Linux" ) + if ( config.m_Keyword == "Linux" ) { - WritePGItem( "BuildCommandLine", cIt->m_ProjectBuildCommand ); - WritePGItem( "ReBuildCommandLine", cIt->m_ProjectRebuildCommand ); - WritePGItem( "CleanCommandLine", cIt->m_ProjectCleanCommand ); + WritePGItem( "BuildCommandLine", config.m_ProjectBuildCommand ); + WritePGItem( "ReBuildCommandLine", config.m_ProjectRebuildCommand ); + WritePGItem( "CleanCommandLine", config.m_ProjectCleanCommand ); } else { - WritePGItem( "NMakeBuildCommandLine", cIt->m_ProjectBuildCommand ); - WritePGItem( "NMakeReBuildCommandLine", cIt->m_ProjectRebuildCommand ); - WritePGItem( "NMakeCleanCommandLine", cIt->m_ProjectCleanCommand ); + WritePGItem( "NMakeBuildCommandLine", config.m_ProjectBuildCommand ); + WritePGItem( "NMakeReBuildCommandLine", config.m_ProjectRebuildCommand ); + WritePGItem( "NMakeCleanCommandLine", config.m_ProjectCleanCommand ); } - WritePGItem( "NMakeOutput", cIt->m_Output ); + WritePGItem( "NMakeOutput", config.m_Output ); const ObjectListNode * oln = nullptr; - if ( cIt->m_PreprocessorDefinitions.IsEmpty() || cIt->m_IncludeSearchPath.IsEmpty() ) + if ( config.m_PreprocessorDefinitions.IsEmpty() || config.m_IncludeSearchPath.IsEmpty() ) { - oln = ProjectGeneratorBase::FindTargetForIntellisenseInfo( cIt->m_TargetNode ); + oln = ProjectGeneratorBase::FindTargetForIntellisenseInfo( config.m_TargetNode ); } - if ( cIt->m_PreprocessorDefinitions.IsEmpty() == false ) + if ( config.m_PreprocessorDefinitions.IsEmpty() == false ) { - WritePGItem( "NMakePreprocessorDefinitions", cIt->m_PreprocessorDefinitions ); + WritePGItem( "NMakePreprocessorDefinitions", config.m_PreprocessorDefinitions ); } else { @@ -334,9 +326,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile { ProjectGeneratorBase::ExtractIncludePaths( oln->GetCompilerOptions(), includePaths, forceIncludes, false ); } - if ( cIt->m_IncludeSearchPath.IsEmpty() == false ) + if ( config.m_IncludeSearchPath.IsEmpty() == false ) { - WritePGItem( "NMakeIncludeSearchPath", cIt->m_IncludeSearchPath ); + WritePGItem( "NMakeIncludeSearchPath", config.m_IncludeSearchPath ); } else if ( oln ) { @@ -351,9 +343,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile ProjectGeneratorBase::ConcatIntellisenseOptions( includePaths, includePathsStr, nullptr, ";" ); WritePGItem( "NMakeIncludeSearchPath", includePathsStr ); } - if ( cIt->m_ForcedIncludes.IsEmpty() == false ) + if ( config.m_ForcedIncludes.IsEmpty() == false ) { - WritePGItem( "NMakeForcedIncludes", cIt->m_ForcedIncludes ); + WritePGItem( "NMakeForcedIncludes", config.m_ForcedIncludes ); } else if ( oln ) { @@ -368,11 +360,11 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile ProjectGeneratorBase::ConcatIntellisenseOptions( forceIncludes, forceIncludePathsStr, nullptr, ";" ); WritePGItem( "NMakeForcedIncludes", forceIncludePathsStr ); } - WritePGItem( "NMakeAssemblySearchPath", cIt->m_AssemblySearchPath ); - WritePGItem( "NMakeForcedUsingAssemblies", cIt->m_ForcedUsingAssemblies ); - if ( cIt->m_AdditionalOptions.IsEmpty() == false ) + WritePGItem( "NMakeAssemblySearchPath", config.m_AssemblySearchPath ); + WritePGItem( "NMakeForcedUsingAssemblies", config.m_ForcedUsingAssemblies ); + if ( config.m_AdditionalOptions.IsEmpty() == false ) { - WritePGItem( "AdditionalOptions", cIt->m_AdditionalOptions ); + WritePGItem( "AdditionalOptions", config.m_AdditionalOptions ); } else { @@ -385,44 +377,43 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile WritePGItem( "AdditionalOptions", additionalOptionsStr ); } } - WritePGItem( "Xbox360DebuggerCommand", cIt->m_Xbox360DebuggerCommand ); - WritePGItem( "DebuggerFlavor", cIt->m_DebuggerFlavor ); - WritePGItem( "AumidOverride", cIt->m_AumidOverride ); - WritePGItem( "LocalDebuggerWorkingDirectory", cIt->m_LocalDebuggerWorkingDirectory ); - WritePGItem( "IntDir", cIt->m_IntermediateDirectory ); - WritePGItem( "OutDir", cIt->m_OutputDirectory ); - WritePGItem( "PackagePath", cIt->m_PackagePath ); - WritePGItem( "AdditionalSymbolSearchPaths", cIt->m_AdditionalSymbolSearchPaths ); - WritePGItem( "LayoutDir", cIt->m_LayoutDir ); - WritePGItem( "LayoutExtensionFilter", cIt->m_LayoutExtensionFilter ); - WritePGItem( "RemoteDebuggerCommand", cIt->m_RemoteDebuggerCommand ); - WritePGItem( "RemoteDebuggerCommandArguments", cIt->m_RemoteDebuggerCommandArguments ); - WritePGItem( "RemoteDebuggerWorkingDirectory", cIt->m_RemoteDebuggerWorkingDirectory ); + WritePGItem( "Xbox360DebuggerCommand", config.m_Xbox360DebuggerCommand ); + WritePGItem( "DebuggerFlavor", config.m_DebuggerFlavor ); + WritePGItem( "AumidOverride", config.m_AumidOverride ); + WritePGItem( "LocalDebuggerWorkingDirectory", config.m_LocalDebuggerWorkingDirectory ); + WritePGItem( "IntDir", config.m_IntermediateDirectory ); + WritePGItem( "OutDir", config.m_OutputDirectory ); + WritePGItem( "PackagePath", config.m_PackagePath ); + WritePGItem( "AdditionalSymbolSearchPaths", config.m_AdditionalSymbolSearchPaths ); + WritePGItem( "LayoutDir", config.m_LayoutDir ); + WritePGItem( "LayoutExtensionFilter", config.m_LayoutExtensionFilter ); + WritePGItem( "RemoteDebuggerCommand", config.m_RemoteDebuggerCommand ); + WritePGItem( "RemoteDebuggerCommandArguments", config.m_RemoteDebuggerCommandArguments ); + WritePGItem( "RemoteDebuggerWorkingDirectory", config.m_RemoteDebuggerWorkingDirectory ); Write( " \n" ); } } // ItemDefinition Groups { - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF(" \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); + WriteF(" \n", config.m_Config.Get(), config.m_Platform.Get() ); Write( " \n" ); - if ( !cIt->m_BuildLogFile.IsEmpty() ) + if ( !config.m_BuildLogFile.IsEmpty() ) { - WritePGItem( "Path", cIt->m_BuildLogFile ); + WritePGItem( "Path", config.m_BuildLogFile ); } else { Write( " \n" ); } Write( " \n" ); - if ( ( !cIt->m_DeploymentType.IsEmpty() ) || ( !cIt->m_DeploymentFiles.IsEmpty() ) ) + if ( !config.m_DeploymentType.IsEmpty() || !config.m_DeploymentFiles.IsEmpty() ) { Write( " \n" ); - WritePGItem( "DeploymentType", cIt->m_DeploymentType ); - WritePGItem( "DeploymentFiles", cIt->m_DeploymentFiles ); + WritePGItem( "DeploymentType", config.m_DeploymentType ); + WritePGItem( "DeploymentFiles", config.m_DeploymentFiles ); Write( " \n" ); } Write( " \n" ); @@ -591,10 +582,8 @@ void VSProjectGenerator::WritePGItem( const char * xmlTag, const AString & value void VSProjectGenerator::GetFolderPath( const AString & fileName, AString & folder ) const { ASSERT( m_FilePathsCanonicalized ); - const AString * const bEnd = m_BasePaths.End(); - for ( const AString * bIt = m_BasePaths.Begin(); bIt != bEnd; ++bIt ) + for ( const AString & basePath : m_BasePaths ) { - const AString & basePath = *bIt; if ( fileName.BeginsWithI( basePath ) ) { const char * begin = fileName.Get() + basePath.GetLength(); From fb022290375beb9476d2db94696deb0c188ae7be Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 13 Aug 2023 12:50:25 +0930 Subject: [PATCH 073/129] [Improvement] VXCProject exposes AndroidApkLocation for Android Game Development Extension [Note] DB Version changed --- .../Tools/FBuild/Documentation/docs/functions/vcxproject.html | 4 ++++ Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 2 +- Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp | 1 + Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h | 1 + Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp | 4 ++++ 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html b/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html index b6cea32bd..ef830db49 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html @@ -135,6 +135,7 @@

VCXProject

.LinuxProjectType // (optional) Set the project type GUID for Linux .PackagePath, // (optional) Path to package to debug .AdditionalSymbolSearchPaths, // (optional) Additional symbol search paths for debugging +.AndroidApkLocation // (optional) Location of APK for Android Game Development Extension // Misc .PlatformToolset // (optional) Specify PlatformToolset. @@ -574,6 +575,9 @@

VCXProject

AdditionalSymbolSearchPaths - String - (Optional)

Path to additional symbols to be used when debugging.

+

AndroidApkLocation - String - (Optional)

+

Location of APK for Android Game Development Extension

+

PlatformToolset - String - (Optional)

Specify PlatformToolset.

diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index 14eadd3ee..cee3065ba 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -64,7 +64,7 @@ class NodeGraphHeader } inline ~NodeGraphHeader() = default; - enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 170 }; + enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 171 }; bool IsValid() const; bool IsCompatibleVersion() const { return m_Version == NODE_GRAPH_CURRENT_VERSION; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp index 1c8afe2d4..7dd2eb9d6 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp @@ -69,6 +69,7 @@ REFLECT_STRUCT_BEGIN_BASE( VSProjectConfigBase ) REFLECT( m_LinuxProjectType, "LinuxProjectType", MetaInheritFromOwner() + MetaOptional() ) REFLECT( m_PackagePath, "PackagePath", MetaInheritFromOwner() + MetaOptional() ) REFLECT( m_AdditionalSymbolSearchPaths, "AdditionalSymbolSearchPaths", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidApkLocation, "AndroidApkLocation", MetaInheritFromOwner() + MetaOptional() ) REFLECT_END( VSProjectConfigBase ) REFLECT_STRUCT_BEGIN( VSProjectConfig, VSProjectConfigBase, MetaNone() ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h index 82e44630f..f63301957 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h @@ -59,6 +59,7 @@ class VSProjectConfigBase : public Struct AString m_LinuxProjectType; AString m_PackagePath; AString m_AdditionalSymbolSearchPaths; + AString m_AndroidApkLocation; }; // VSProjectConfig diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp index 1ead2f2b2..7268077e0 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp @@ -297,6 +297,10 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile WritePGItem( "NMakeReBuildCommandLine", config.m_ProjectRebuildCommand ); WritePGItem( "NMakeCleanCommandLine", config.m_ProjectCleanCommand ); } + if ( !config.m_AndroidApkLocation.IsEmpty() ) + { + WritePGItem( "AndroidApkLocation", config.m_AndroidApkLocation ); + } WritePGItem( "NMakeOutput", config.m_Output ); const ObjectListNode * oln = nullptr; From 406f5dd0fa419720e50243524a809ee2d82b13af Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 13 Aug 2023 12:52:16 +0930 Subject: [PATCH 074/129] ThreadPool: Extend lifetime of ThreadPool - Create earlier and destroy later - This allows build work to start slightly earlier and facilitates using the ThreadPool for some additional things in the future --- Code/Tools/FBuild/FBuildCore/FBuild.cpp | 11 ++++++++++- Code/Tools/FBuild/FBuildCore/FBuild.h | 2 ++ Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp | 9 ++------- Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h | 3 +-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/FBuild.cpp b/Code/Tools/FBuild/FBuildCore/FBuild.cpp index bae1ee809..2412e9c17 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuild.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuild.cpp @@ -34,6 +34,7 @@ #include "Core/Mem/SmallBlockAllocator.h" #include "Core/Process/Atomic.h" #include "Core/Process/SystemMutex.h" +#include "Core/Process/ThreadPool.h" #include "Core/Profile/Profile.h" #include "Core/Strings/AStackString.h" #include "Core/Tracing/Tracing.h" @@ -78,6 +79,12 @@ FBuild::FBuild( const FBuildOptions & options ) // store all user provided options m_Options = options; + // Create ThreadPool + if ( m_Options.m_NumWorkerThreads > 0 ) + { + m_ThreadPool = FNEW( ThreadPool( m_Options.m_NumWorkerThreads ) ); + } + // track the old working dir to restore if modified (mainly for unit tests) VERIFY( FileIO::GetCurrentDir( m_OldWorkingDir ) ); @@ -129,6 +136,8 @@ FBuild::~FBuild() { FDELETE( &BuildProfiler::Get() ); } + + FDELETE m_ThreadPool; } // Initialize @@ -373,7 +382,7 @@ void FBuild::SaveDependencyGraph( MemoryStream & stream, const char* nodeGraphDB AtomicStoreRelaxed( &s_AbortBuild, false ); // allow multiple runs in same process // create worker threads - m_JobQueue = FNEW( JobQueue( m_Options.m_NumWorkerThreads ) ); + m_JobQueue = FNEW( JobQueue( m_Options.m_NumWorkerThreads, m_ThreadPool ) ); // create the connection management system if needed // (must be after JobQueue is created) diff --git a/Code/Tools/FBuild/FBuildCore/FBuild.h b/Code/Tools/FBuild/FBuildCore/FBuild.h index d7028434f..2898fb52d 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuild.h +++ b/Code/Tools/FBuild/FBuildCore/FBuild.h @@ -27,6 +27,7 @@ class MemoryStream; class JobQueue; class Node; class NodeGraph; +class ThreadPool; // FBuild //------------------------------------------------------------------------------ @@ -122,6 +123,7 @@ class FBuild : public Singleton< FBuild > static volatile bool s_AbortBuild; // -fastcancel - TODO:C merge with StopBuild NodeGraph * m_DependencyGraph; + ThreadPool * m_ThreadPool = nullptr; JobQueue * m_JobQueue; mutable Mutex m_ClientLifetimeMutex; Client * m_Client; // manage connections to worker servers diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp index def244ff0..994ffa201 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp @@ -142,7 +142,7 @@ Job * JobSubQueue::RemoveJob() // CONSTRUCTOR //------------------------------------------------------------------------------ -JobQueue::JobQueue( uint32_t numWorkerThreads ) : +JobQueue::JobQueue( uint32_t numWorkerThreads, ThreadPool * threadPool ) : m_NumLocalJobsActive( 0 ), m_DistributableJobs_Available( 1024, true ), m_DistributableJobs_InProgress( 1024, true ), @@ -163,9 +163,6 @@ JobQueue::JobQueue( uint32_t numWorkerThreads ) : if ( numWorkerThreads > 0 ) { - // Create thread pool - m_ThreadPool = FNEW( ThreadPool( numWorkerThreads ) ); - // Create a job to run on each thread for ( uint32_t i = 0; i < numWorkerThreads; ++i ) { @@ -173,7 +170,7 @@ JobQueue::JobQueue( uint32_t numWorkerThreads ) : // (the "main" thread is considered 0) const uint16_t threadIndex = static_cast( i + 1 ); WorkerThread * wt = FNEW( WorkerThread( threadIndex ) ); - wt->Init( m_ThreadPool ); + wt->Init( threadPool ); m_Workers.Append( wt ); } } @@ -218,8 +215,6 @@ JobQueue::~JobQueue() ASSERT( m_CompletedJobs.IsEmpty() ); ASSERT( m_CompletedJobsFailed.IsEmpty() ); ASSERT( Job::GetTotalLocalDataMemoryUsage() == 0 ); - - FDELETE m_ThreadPool; } // SignalStopWorkers (Main Thread) diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h index e75cc9c3c..4a3e4f80d 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h @@ -45,7 +45,7 @@ class JobSubQueue class JobQueue : public Singleton< JobQueue > { public: - explicit JobQueue( uint32_t numWorkerThreads ); + explicit JobQueue( uint32_t numWorkerThreads, ThreadPool * threadPool ); ~JobQueue(); // main thread calls these @@ -118,7 +118,6 @@ class JobQueue : public Singleton< JobQueue > Array< Job * > m_CompletedJobs2; Array< Job * > m_CompletedJobsFailed2; - ThreadPool * m_ThreadPool = nullptr; Array< WorkerThread * > m_Workers; }; From 7626b649fdae3f50effdd7cf031cc4eb5504059b Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:19:04 +0930 Subject: [PATCH 075/129] Update integration files for new properties --- Code/Tools/FBuild/Integration/notepad++markup.xml | 2 +- Code/Tools/FBuild/Integration/usertype.dat | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Code/Tools/FBuild/Integration/notepad++markup.xml b/Code/Tools/FBuild/Integration/notepad++markup.xml index eba899a61..696675c72 100644 --- a/Code/Tools/FBuild/Integration/notepad++markup.xml +++ b/Code/Tools/FBuild/Integration/notepad++markup.xml @@ -25,7 +25,7 @@ Alias CSAssembly Compiler Copy CopyDir DLL Error Exec Executable ForEach If Library ListDependencies ObjectList Print RemoveDir Settings Test TextFile Unity Using VCXProject VSProjectExternal VSSolution XCodeProject - AdditionalOptions AdditionalSymbolSearchPaths AllowCaching AllowDistribution AllowResponseFile ApplicationEnvironment ApplicationType ApplicationTypeRevision AssemblySearchPath AumidOverride BaseProjectConfig BaseSolutionConfig BuildLogFile CachePath CachePathMountPoint CachePluginDLL CachePluginDLLConfig ClangFixupUnity_Disable ClangGCCUpdateXLanguageArg ClangRewriteIncludes Compiler CompilerFamily CompilerForceUsing CompilerInputAllowNoFiles CompilerInputExcludePath CompilerInputExcludePattern CompilerInputExcludedFiles CompilerInputFile CompilerInputFiles CompilerInputFilesRoot CompilerInputObjectLists CompilerInputPath CompilerInputPathRecurse CompilerInputPattern CompilerInputUnity CompilerOptions CompilerOptionsDeoptimized CompilerOutput CompilerOutputExtension CompilerOutputKeepBaseExtension CompilerOutputPath CompilerOutputPrefix CompilerReferences Condition Config CustomEnvironmentVariables DebuggerFlavor DefaultLanguage DeoptimizeWritableFiles DeoptimizeWritableFilesWithToken Dependencies DeploymentFiles DeploymentType Dest DistributableJobMemoryLimitMiB Environment ExecAlways ExecAlwaysShowOutput ExecArguments ExecExecutable ExecInput ExecInputExcludePath ExecInputExcludePattern ExecInputExcludedFiles ExecInputPath ExecInputPathRecurse ExecInputPattern ExecOutput ExecReturnCode ExecUseStdOutAsOutput ExecWorkingDir Executable ExecutableRootPath ExternalProjectPath ExtraFiles FileType ForceResponseFile ForcedIncludes ForcedUsingAssemblies Hidden IncludeSearchPath IntermediateDirectory Items Keyword LayoutDir LayoutExtensionFilter Librarian LibrarianAdditionalInputs LibrarianAllowResponseFile LibrarianForceResponseFile LibrarianOptions LibrarianOutput LibrarianType Libraries Libraries2 Linker LinkerAllowResponseFile LinkerAssemblyResources LinkerForceResponseFile LinkerLinkObjects LinkerOptions LinkerOutput LinkerStampExe LinkerStampExeArgs LinkerType LinuxProjectType LocalDebuggerCommand LocalDebuggerCommandArguments LocalDebuggerEnvironment LocalDebuggerWorkingDirectory Output OutputDirectory PCHInputFile PCHObjectFileName PCHOptions PCHOutputFile PackagePath Path Pattern Patterns Platform PlatformToolset PreBuildDependencies Preprocessor PreprocessorDefinitions PreprocessorOptions Project ProjectAllowedFileExtensions ProjectBasePath ProjectBuildCommand ProjectCleanCommand ProjectConfigs ProjectFileTypes ProjectFiles ProjectFilesToExclude ProjectGuid ProjectInputPaths ProjectInputPathsExclude ProjectInputPathsRecurse ProjectOutput ProjectPatternToExclude ProjectProjectImports ProjectProjectReferences ProjectRebuildCommand ProjectReferences ProjectSccEntrySAK ProjectTypeGuid Projects RemoteDebuggerCommand RemoteDebuggerCommandArguments RemoteDebuggerWorkingDirectory RemoveExcludeFiles RemoveExcludePaths RemovePaths RemovePathsRecurse RemovePatterns RootNamespace SimpleDistributionMode SolutionBuildProject SolutionConfig SolutionConfigs SolutionDependencies SolutionDeployProjects SolutionFolders SolutionMinimumVisualStudioVersion SolutionOutput SolutionPlatform SolutionProjects SolutionVisualStudioVersion Source SourceExcludePaths SourceMapping_Experimental SourcePaths SourcePathsPattern SourcePathsRecurse Target TargetLinuxPlatform Targets TestAlwaysShowOutput TestArguments TestExecutable TestInput TestInputExcludePath TestInputExcludePattern TestInputExcludedFiles TestInputPath TestInputPathRecurse TestInputPattern TestOutput TestTimeOut TestWorkingDir TextFileAlways TextFileInputStrings TextFileOutput UnityInputExcludePath UnityInputExcludePattern UnityInputExcludedFiles UnityInputFiles UnityInputIsolateListFile UnityInputIsolateWritableFiles UnityInputIsolateWritableFilesLimit UnityInputIsolatedFiles UnityInputObjectLists UnityInputPath UnityInputPathRecurse UnityInputPattern UnityNumFiles UnityOutputPath UnityOutputPattern UnityPCH UseLightCache_Experimental UseRelativePaths_Experimental VS2012EnumBugFix WorkerConnectionLimit Workers XCodeBaseSDK XCodeBuildToolArgs XCodeBuildToolPath XCodeBuildWorkingDir XCodeCommandLineArguments XCodeCommandLineArgumentsDisabled XCodeDebugWorkingDir XCodeDocumentVersioning XCodeIphoneOSDeploymentTarget XCodeOrganizationName Xbox360DebuggerCommand + AdditionalOptions AdditionalSymbolSearchPaths AllowCaching AllowDistribution AllowResponseFile AndroidApkLocation ApplicationEnvironment ApplicationType ApplicationTypeRevision AssemblySearchPath AumidOverride BaseProjectConfig BaseSolutionConfig BuildLogFile CachePath CachePathMountPoint CachePluginDLL CachePluginDLLConfig ClangFixupUnity_Disable ClangGCCUpdateXLanguageArg ClangRewriteIncludes Compiler CompilerFamily CompilerForceUsing CompilerInputAllowNoFiles CompilerInputExcludePath CompilerInputExcludePattern CompilerInputExcludedFiles CompilerInputFile CompilerInputFiles CompilerInputFilesRoot CompilerInputObjectLists CompilerInputPath CompilerInputPathRecurse CompilerInputPattern CompilerInputUnity CompilerOptions CompilerOptionsDeoptimized CompilerOutput CompilerOutputExtension CompilerOutputKeepBaseExtension CompilerOutputPath CompilerOutputPrefix CompilerReferences Condition Config CustomEnvironmentVariables DebuggerFlavor DefaultLanguage DeoptimizeWritableFiles DeoptimizeWritableFilesWithToken Dependencies DeploymentFiles DeploymentType Dest DistributableJobMemoryLimitMiB Environment ExecAlways ExecAlwaysShowOutput ExecArguments ExecExecutable ExecInput ExecInputExcludePath ExecInputExcludePattern ExecInputExcludedFiles ExecInputPath ExecInputPathRecurse ExecInputPattern ExecOutput ExecReturnCode ExecUseStdOutAsOutput ExecWorkingDir Executable ExecutableRootPath ExternalProjectPath ExtraFiles FileType ForceResponseFile ForcedIncludes ForcedUsingAssemblies Hidden IncludeSearchPath IntermediateDirectory Items Keyword LayoutDir LayoutExtensionFilter Librarian LibrarianAdditionalInputs LibrarianAllowResponseFile LibrarianForceResponseFile LibrarianOptions LibrarianOutput LibrarianType Libraries Libraries2 Linker LinkerAllowResponseFile LinkerAssemblyResources LinkerForceResponseFile LinkerLinkObjects LinkerOptions LinkerOutput LinkerStampExe LinkerStampExeArgs LinkerType LinuxProjectType LocalDebuggerCommand LocalDebuggerCommandArguments LocalDebuggerEnvironment LocalDebuggerWorkingDirectory Output OutputDirectory PCHInputFile PCHObjectFileName PCHOptions PCHOutputFile PackagePath Path Pattern Patterns Platform PlatformToolset PreBuildDependencies Preprocessor PreprocessorDefinitions PreprocessorOptions Project ProjectAllowedFileExtensions ProjectBasePath ProjectBuildCommand ProjectCleanCommand ProjectConfigs ProjectFileTypes ProjectFiles ProjectFilesToExclude ProjectGuid ProjectInputPaths ProjectInputPathsExclude ProjectInputPathsRecurse ProjectOutput ProjectPatternToExclude ProjectProjectImports ProjectProjectReferences ProjectRebuildCommand ProjectReferences ProjectSccEntrySAK ProjectTypeGuid Projects RemoteDebuggerCommand RemoteDebuggerCommandArguments RemoteDebuggerWorkingDirectory RemoveExcludeFiles RemoveExcludePaths RemovePaths RemovePathsRecurse RemovePatterns RootNamespace SimpleDistributionMode SolutionBuildProject SolutionConfig SolutionConfigs SolutionDependencies SolutionDeployProjects SolutionFolders SolutionMinimumVisualStudioVersion SolutionOutput SolutionPlatform SolutionProjects SolutionVisualStudioVersion Source SourceExcludePaths SourceMapping_Experimental SourcePaths SourcePathsPattern SourcePathsRecurse Target TargetLinuxPlatform Targets TestAlwaysShowOutput TestArguments TestExecutable TestInput TestInputExcludePath TestInputExcludePattern TestInputExcludedFiles TestInputPath TestInputPathRecurse TestInputPattern TestOutput TestTimeOut TestWorkingDir TextFileAlways TextFileInputStrings TextFileOutput UnityInputExcludePath UnityInputExcludePattern UnityInputExcludedFiles UnityInputFiles UnityInputIsolateListFile UnityInputIsolateWritableFiles UnityInputIsolateWritableFilesLimit UnityInputIsolatedFiles UnityInputObjectLists UnityInputPath UnityInputPathRecurse UnityInputPattern UnityNumFiles UnityOutputPath UnityOutputPattern UnityPCH UseLightCache_Experimental UseRelativePaths_Experimental VS2012EnumBugFix WorkerConnectionLimit Workers XCodeBaseSDK XCodeBuildToolArgs XCodeBuildToolPath XCodeBuildWorkingDir XCodeCommandLineArguments XCodeCommandLineArgumentsDisabled XCodeDebugWorkingDir XCodeDocumentVersioning XCodeIphoneOSDeploymentTarget XCodeOrganizationName Xbox360DebuggerCommand ) %1 %2 %3 diff --git a/Code/Tools/FBuild/Integration/usertype.dat b/Code/Tools/FBuild/Integration/usertype.dat index 4fa7d62c7..aaa4dbb4d 100644 --- a/Code/Tools/FBuild/Integration/usertype.dat +++ b/Code/Tools/FBuild/Integration/usertype.dat @@ -45,6 +45,7 @@ AdditionalSymbolSearchPaths AllowCaching AllowDistribution AllowResponseFile +AndroidApkLocation ApplicationEnvironment ApplicationType ApplicationTypeRevision From 063778fa902c14bc0fbec8a9f98a0d3797abc600 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:20:01 +0930 Subject: [PATCH 076/129] v1.11 --- .../FBuild/Documentation/docs/changelog.html | 27 +++++++++++++++++ .../FBuild/Documentation/docs/download.html | 30 ++++++++++++------- .../Tools/FBuild/Documentation/docs/home.html | 7 ++--- Code/Tools/FBuild/FBuildCore/FBuildVersion.h | 4 +-- 4 files changed, 52 insertions(+), 16 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/changelog.html b/Code/Tools/FBuild/Documentation/docs/changelog.html index 2486d3c9f..531b4808f 100644 --- a/Code/Tools/FBuild/Documentation/docs/changelog.html +++ b/Code/Tools/FBuild/Documentation/docs/changelog.html @@ -26,6 +26,33 @@

Changelog

+
+ v1.11 (19th-Aug-2023) +
+
+ Notes +
    +
  • DB Version Changed : clean build will occur
    • +
+ Fixes +
    +
  • [Linux] Robustify file copy if sendfile isn't supported (Thanks to Nick Krecklow)
    • +
+ Improvements +
    +
  • Reduce cost of queueing jobs during graph traversal
    • +
  • Improve LightCache performance considerably for many CPU core systems
    • +
  • Directory listings with excluded folders optimized significantly
    • +
  • All directory listings optimized slightly
    • +
  • Database loading and various lookup performance improved slightly
    • +
  • Improve database load time slightly
    • +
  • Error #1100 emits location of previous declaration
    • +
  • -dbfile arg can explicitly set dependency db file location (Thanks to Troels Gram)
    • +
  • [Linux] Improve detection of max args length using _SC_ARG_MAX (Thanks to Domain)
    • +
  • VXCProject exposes AndroidApkLocation for Android Game Development Extension
    • +
+
+
v1.10 (28th-May-2023)
diff --git a/Code/Tools/FBuild/Documentation/docs/download.html b/Code/Tools/FBuild/Documentation/docs/download.html index ad96b027d..b871b67a9 100644 --- a/Code/Tools/FBuild/Documentation/docs/download.html +++ b/Code/Tools/FBuild/Documentation/docs/download.html @@ -39,22 +39,22 @@

Download

- - - - - + + + + +
VersionWindowsOS XLinuxSource
v1.10x64x64 & ARMx64Zip | - GitHub
v1.11x64x64 & ARMx64Zip | + GitHub
Syntax Highlighting
@@ -87,6 +87,16 @@

Editor Integration

+ + + + + + + + + diff --git a/Code/Tools/FBuild/Documentation/docs/home.html b/Code/Tools/FBuild/Documentation/docs/home.html index e766d5477..d55993e9d 100644 --- a/Code/Tools/FBuild/Documentation/docs/home.html +++ b/Code/Tools/FBuild/Documentation/docs/home.html @@ -38,14 +38,13 @@
- FASTBuild v1.10 released!  (28-May-2023) + FASTBuild v1.11 released!  (19-Aug-2023)
An updated version of FASTBuild can now be downloaded, featuring:
    -
  • Native support for Apple Silicon (ARM) Macs, significantly improving performance
    • -
  • Various performance enhancements
    • -
  • Some misc bug fixes and reliability improvements
    • +
  • Several general performance enhancements
    • +
  • Various small bug fixes and reliability improvements
  • Details...
diff --git a/Code/Tools/FBuild/FBuildCore/FBuildVersion.h b/Code/Tools/FBuild/FBuildCore/FBuildVersion.h index d227ef6a4..3c4c21386 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildVersion.h +++ b/Code/Tools/FBuild/FBuildCore/FBuildVersion.h @@ -4,8 +4,8 @@ // Defines //------------------------------------------------------------------------------ -#define FBUILD_VERSION_STRING "v1.10" -#define FBUILD_VERSION (uint32_t)110 +#define FBUILD_VERSION_STRING "v1.11" +#define FBUILD_VERSION (uint32_t)111 #if defined( __WINDOWS__ ) #define FBUILD_VERSION_PLATFORM "Windows" #elif defined( __APPLE__ ) From 147c5ff7b7ade23d05da3498eb8c280982eeff7a Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 12:25:07 +0930 Subject: [PATCH 077/129] Integrate dev -> main for v1.11 (#994) * Update dev CI to use v1.09 * Update OSX CI to use macos11 - github reports that 10.15 is deprecated * Switch CI "checkout" action to v3 - update to avoid deprecation warnings about the old action using nodejs 12 * Add VS2022 to CI coverage * Remove VS2022 from CI coverage - some configuration issues need to be resolved * Add xxHash v0.8.1 (not yet used) * xxHash3: Expose xxHash3 64 for future use (2-3x faster than regular xxhash 64) * Threads: Replace most deprecated Thread API usage with newer/safer API - Also activate Thread test that was not enabled * Core: Remove most of deprecated Thread API - use new API where possible - trim legacy API to only what remains in use - make Thread internally directly use what it needs instead of forwarding to the deprecated API * BFFFuzzer: Fix linking with xxHash3 * Various small performance improvements through use of xxHash3 [Improvement] Cache performance improved slightly for check, store and retrieve operations [Improvement] Toolchain management performance improved slightly when toolchain changes [Improvement] Performance of various build steps improved slightly [Improvement] Database validation improved slightly, reducing startup/shutdown time slightly - switch 64bit hashes to xxHash3 which is ~2x-3x faster than xxHash [Note] DB version changed [Note] Cache version changed * BFFVariables: Variables track their origin to facilitate improved errors in the future - We keep a pointer to the BFFToken responsible for the creation of the variable * Fix Linux and MacOS file permissions: (#956) * SetExecutable: preserve existing RW permissions; just add X * SetReadOnly: when removing the flag, remove W entirely; when adding the flag, only set for USR * mkdir: Create directories with 0755 permissions * Post-integration cleanup for Linux/OSX FileIO permissions fixes - minor style fixes * Remove Linux GCC 7 from github actions CI - seems like either GCC7 or the version of ubuntu used causes the github action to be stuck in queue forever * Mutex: Small code cleanup (no functional changes) * Mutex: Add TryLock functionality - speculative lock acquisition that returns immediately on failure * [Fix] Prevent Worker over-requesting jobs when there are multiple Clients * Remove remnants of 32bits support for compiling FASTBuild - FASTBuild has been an exclusively 64bit process for sometime - The processes it can spawn are unaffected * Tidy up architecture defines - use compile built-ins for detection of X64 vs ARM instead of setting those in the bff files - this is necessary to allow multiple architectures to be generated in one compiler invocation for OSX Universal Binaries * OSX: Use Clang12 to build FASTBuild by default * OSX: Simplify Clang12 configs - de-duplicate config common to x64 and ARM * [Improvement][OSX] Add native Apple Silicon support using Universal binaries - combine existing arm and x64 executables together into universal binaries via new "OSX" targets - gated on availability of Clang12 or newer * git CI uses Universal binary targets for OSX * Fix collision errors on chained ObjectLists (#966) * Chained ObjectLists now properly handle sub-directories. * Fix test by increasing number of objects seend and built. * Post-intergation: Chained ObjectList fix * [Improvement] FBuildWorker will restart every 4 hours to improve reliability - this helps ensure intermittent issues don't impact overall worker pool reliability * [Improvment] Remove some unecessary lock contention in FBuildWorker - avoid locking the worker list unnecessarily when not accepting work * Mutex: Add TryMutexHolder to support RAII patterns with speculatively acquired Mutexes * [Improvement] Eliminate various instances of lock contention in FBuildWorker * Fix requested job count becoming negative under heavy load - result of request could come back before incrementing requested job count. Increment must occur before Send(). - problem introduced when removing lock contention (lock would have blocked receiver previously) * [Fix] Fix crash when .CompilerOutputPath is missing but required on ObjectList/Library * Fix docs to mark various function properties as optional that were incorreclty marked as required (#976) * Fix docs to mark various function properties as optional that were incorreclty marked as required * `DLL`/`Executable`: `Libraries2` * `Library`/`ObjectList`: `CompilerOutputPath` * `XCodeProject`: `XCodeOrganizationName` * Re-mark `CompilerOutputPath` as required, un-trim trailing whitespace --------- Co-authored-by: Harrison Ting * Add the "FASTBuild Support" VS Code extension to the "3rd Party Downloads and Extensions" docs (#977) Co-authored-by: Harrison Ting * Update styling for 3rd Party "Editor Integration" links * Worker periodic restart is disabled by default and can be activated via -periodicrestart * v1.10 * Fix documentation errors (Thanks to Troels Gram) - bad brace style noted for -config (arg is required, not optional) - bad hypth style for -continueafterdbmove (args are prefixed with one hyphen) * Add -dbfile option to set dependency database file explicitly (#970) * Add support for -dbfile option to set dependency graph file explicitly * Add DBLocation test to TestGraph suite * Fix test database deletion * Added docs to -dbfile option * Post-integration: -dbfile cleanup - avoid an unnecessary string copy * Cleanup node creation logic: - eliminate most bespoke functions and mostly consolidate around a single CreateNode function * [Improvement] Improve database load time slightly - Avoid double handling of node names - Avoid unnecessary path normalization during loading (paths saved are already normalized) - Simplify node creation by eliminating unnecessary args passing - Remove CreateFileNode (Intergrate special case into regular CreateNode() function. Wliminating this special case entirely is still desirable and abstracting the special case away from callers is a step towards that. * Code cleanup - normalize whitespace [Improvement] Database loading and various lookup performance improved slightly - use xxHash3 instead of CRC32 for node name hashes Revert "[Improvement] Database loading and various lookup performance improved slightly" This reverts commit 775c62f7153b8d3ba1243e526643557979e18ced. * [Improvement] Database loading and various lookup performance improved slightly - use xxHash3 instead of CRC32 for node name hashes * [Improvement] Reduce cost of queueing jobs during graph traversal - avoid re-sorting a list when we can merge sorted lists * Remove unnecessary atomic op generating Job Ids - these are always created on the main thread * Remove unused forceClean param from DoDynamicDependencies * TestExe: Remove redundant test - TestNodeGraph tests basic creation of this node already * [Improvement] Error #1100 emits location of previous declaration - Track the token associated with the creation of targets where possible - When emitting error 1100 emit additional information about the previous declaration if possible * [Improvement] All directory listings optimized slightly - walk each directory only once (instead of once for files and once for folders) - move strings into final destination instead of copying [Improvement] Directory listings with excluded folders optimized significantly - filter directories as the tree is walked instead of after, eliminating recursion into sub-directories * [Improvement] Improve LightCache performance considerably for many CPU core systems - Eliminate a large amount of redundant processing, particularly at the start of builds by extending lock duration to entire processing operation. The slight reduction in parallelism due to collisions is more than offset by the reduced redundant work. In testing, the host processing time was more than halved. - Increase number of buckets by 4x to reduce contention. This further offsets collision impact due to the above, but is also a gain without those changes. - Avoid some string copies by moving results into final destination - Ensure bucket structures are cacheline aligned (typically already the case, but this ensures layout is predictable) * Job: Eliminate non-atomic access to mem usage during an assert * NodeGraph::BuildRecurse: - convert multiple if statements into switch/case - eliminate states that were never used (always replaced) - rename states for clarity Should be functionally unchanged. * Remove Clang 7 configs for FASTBuild on Windows: - we don't need to build FASTBuild itself on Windows with such an old version - the Clang versions FASTBuild can use is unchanged * Remove remnants of support for building FASTBuild with VS pre VS2017: - we haven't needed or maintained that for a long time - this has no impact on which versions FASTBuild can use * Zstd: Prepare Zstd (v1.5.5) for future use * ThreadPool: Create a general purpose ThreadPool - Will allow operations outside of the JobQueue to be parallelized - Set thread names for profiling automatically - Fix unsafe threadname setting in debug builds (pointer to string with unsafe lifetime) * GCC: Add config to build FASTBuild with gcc11 on Linux * Clang: Add config to build FASTBuild with Clang14 on Linux * GCC: Restore accidentally deleted Clang9 config for Linux * Zstd: Fix compilation for Linux - disable use of asm - it's not used on other platforms: it would be worth investigating if this is a signifcant performance gain for our use -case or not * Zstd: Improve compile time (110 CPU seconds -> 50 CPU seconds) - exclude some files we don't need - use Unity to reduce redundant processing * Zstd compression support groundwork: - extend internal helper and tests to support Zstd compression as well as LZ4 * Ztsd: Fix Zstd compilation error - Fix Windows static analysis config - disable asm dependencies on x86_64 OSX like Linux (as with Linux, this needs investigation to understand the performance impact) * JobQueue/JobQueueRemote use ThreadPool: - prepares for some future additional use of the ThreadPool * TestExec: Fix Exec test invoking Build twice on the same FBuild instance - this is not valid and is also inefficient * SmallBlockAllocator: Remove single-threaded mode - while faster, this mode is unsafe and anything so allocation heavy as to be significantly impacted by this can probably be optimized more effectively in other ways * get limit from system configuration (#923) * get limit from system configuration * args not long enough * Post-integration cleanup: Linux args limit detection improvement - minor style fixes - improve/update comments - tweak args limit size for tests * [Fix][Linux] Robustify file copy if sendfile isn't supported (Thanks to Nick Krecklow) - if sendfile fails, copy contents of file manually in 4KiB chunks * VCProjectGenerator: Tidy up loops - use range-based for to simplify some loops * [Improvement] VXCProject exposes AndroidApkLocation for Android Game Development Extension [Note] DB Version changed * ThreadPool: Extend lifetime of ThreadPool - Create earlier and destroy later - This allows build work to start slightly earlier and facilitates using the ThreadPool for some additional things in the future * Update integration files for new properties * v1.11 --------- Co-authored-by: Pete Lewis Co-authored-by: Dandielo Co-authored-by: Harrison Ting Co-authored-by: Harrison Ting Co-authored-by: Troels Gram <28827711+to01z@users.noreply.github.com> Co-authored-by: Domain --- Code/.FASTBuild/HelperFunctions.bff | 2 +- Code/Core/CoreTest/TestMain.cpp | 1 + Code/Core/CoreTest/Tests/TestFileIO.cpp | 4 +- Code/Core/CoreTest/Tests/TestSemaphore.cpp | 2 +- .../Tests/TestSmallBlockAllocator.cpp | 20 +- Code/Core/CoreTest/Tests/TestThreadPool.cpp | 89 + Code/Core/Env/Assert.cpp | 6 +- Code/Core/Env/Env.cpp | 2 +- Code/Core/Env/Types.h | 7 - Code/Core/Env/WindowsHeader.h | 4 +- Code/Core/FileIO/FileIO.cpp | 242 +- Code/Core/FileIO/FileIO.h | 33 + Code/Core/Mem/Mem.cpp | 8 +- Code/Core/Mem/Mem.h | 4 +- Code/Core/Mem/SmallBlockAllocator.cpp | 60 +- Code/Core/Mem/SmallBlockAllocator.h | 13 - Code/Core/Network/Network.cpp | 2 +- Code/Core/Network/TCPConnectionPool.h | 2 +- Code/Core/Process/Process.cpp | 6 +- Code/Core/Process/Thread.cpp | 31 +- Code/Core/Process/ThreadPool.cpp | 173 + Code/Core/Process/ThreadPool.h | 70 + Code/Core/Strings/AString.cpp | 4 +- Code/OSUI/OSWindow.cpp | 2 +- Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff | 1 + .../FBuild/Documentation/docs/changelog.html | 27 + .../FBuild/Documentation/docs/download.html | 30 +- .../docs/functions/vcxproject.html | 4 + .../Tools/FBuild/Documentation/docs/home.html | 7 +- .../FBuild/Documentation/docs/options.html | 18 +- Code/Tools/FBuild/FBuild/FBuild.bff | 3 +- Code/Tools/FBuild/FBuild/Main.cpp | 2 +- .../FBuildCore/BFF/BFFBooleanExpParser.cpp | 4 +- .../Tools/FBuild/FBuildCore/BFF/BFFParser.cpp | 4 +- .../FBuild/FBuildCore/BFF/BFFStackFrame.cpp | 2 +- .../FBuild/FBuildCore/BFF/BFFStackFrame.h | 2 +- .../FBuildCore/BFF/Functions/Function.cpp | 50 +- .../FBuildCore/BFF/Functions/Function.h | 5 +- .../FBuildCore/BFF/Functions/FunctionCopy.cpp | 11 +- .../FBuildCore/BFF/Functions/FunctionIf.cpp | 4 +- .../BFF/Functions/FunctionObjectList.cpp | 4 +- .../BFF/Functions/FunctionObjectList.h | 2 +- .../BFF/Functions/FunctionRemoveDir.cpp | 7 +- .../BFF/Functions/FunctionSettings.cpp | 7 +- .../FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp | 6 +- .../FBuild/FBuildCore/Cache/LightCache.cpp | 68 +- Code/Tools/FBuild/FBuildCore/Error.cpp | 9 +- Code/Tools/FBuild/FBuildCore/Error.h | 3 +- .../ExeDrivers/Compiler/CompilerDriverBase.h | 2 +- .../ExeDrivers/Compiler/CompilerDriver_CL.cpp | 4 +- .../ExeDrivers/Compiler/CompilerDriver_CL.h | 2 +- Code/Tools/FBuild/FBuildCore/FBuild.cpp | 45 +- Code/Tools/FBuild/FBuildCore/FBuild.h | 2 + Code/Tools/FBuild/FBuildCore/FBuildCore.bff | 1 + .../Tools/FBuild/FBuildCore/FBuildOptions.cpp | 24 +- Code/Tools/FBuild/FBuildCore/FBuildOptions.h | 1 + Code/Tools/FBuild/FBuildCore/FBuildVersion.h | 4 +- .../FBuild/FBuildCore/Graph/AliasNode.cpp | 3 +- Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp | 6 +- Code/Tools/FBuild/FBuildCore/Graph/CSNode.h | 2 +- .../FBuild/FBuildCore/Graph/CompilerNode.cpp | 4 +- .../FBuild/FBuildCore/Graph/CopyDirNode.cpp | 10 +- .../FBuild/FBuildCore/Graph/CopyDirNode.h | 2 +- .../FBuild/FBuildCore/Graph/CopyFileNode.cpp | 2 +- .../FBuild/FBuildCore/Graph/Dependencies.cpp | 2 +- .../FBuildCore/Graph/DirectoryListNode.cpp | 144 +- .../FBuildCore/Graph/DirectoryListNode.h | 2 +- .../FBuild/FBuildCore/Graph/ExecNode.cpp | 8 +- Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h | 2 +- .../FBuild/FBuildCore/Graph/FileNode.cpp | 16 +- Code/Tools/FBuild/FBuildCore/Graph/FileNode.h | 2 +- .../FBuild/FBuildCore/Graph/LibraryNode.cpp | 6 +- .../FBuild/FBuildCore/Graph/LibraryNode.h | 2 +- .../FBuild/FBuildCore/Graph/LinkerNode.cpp | 8 +- .../FBuild/FBuildCore/Graph/LinkerNode.h | 2 +- .../FBuildCore/Graph/ListDependenciesNode.cpp | 2 +- Code/Tools/FBuild/FBuildCore/Graph/Node.cpp | 93 +- Code/Tools/FBuild/FBuildCore/Graph/Node.h | 22 +- .../FBuild/FBuildCore/Graph/NodeGraph.cpp | 559 +- .../Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 44 +- .../FBuild/FBuildCore/Graph/NodeProxy.cpp | 5 +- .../Tools/FBuild/FBuildCore/Graph/NodeProxy.h | 2 +- .../FBuildCore/Graph/ObjectListNode.cpp | 20 +- .../FBuild/FBuildCore/Graph/ObjectListNode.h | 4 +- .../FBuild/FBuildCore/Graph/ObjectNode.cpp | 29 +- .../FBuild/FBuildCore/Graph/ObjectNode.h | 2 +- .../FBuild/FBuildCore/Graph/RemoveDirNode.cpp | 3 +- .../Tools/FBuild/FBuildCore/Graph/SLNNode.cpp | 5 +- .../FBuild/FBuildCore/Graph/SettingsNode.cpp | 6 +- .../FBuild/FBuildCore/Graph/TestNode.cpp | 6 +- Code/Tools/FBuild/FBuildCore/Graph/TestNode.h | 2 +- .../FBuild/FBuildCore/Graph/TextFileNode.cpp | 4 +- .../FBuild/FBuildCore/Graph/UnityNode.cpp | 4 +- .../FBuildCore/Graph/VCXProjectNode.cpp | 1 + .../FBuild/FBuildCore/Graph/VCXProjectNode.h | 1 + .../FBuildCore/Graph/VSProjectBaseNode.cpp | 3 +- .../Graph/VSProjectExternalNode.cpp | 4 +- .../FBuildCore/Graph/XCodeProjectNode.cpp | 3 +- Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp | 8 +- .../FBuild/FBuildCore/Helpers/Compressor.cpp | 105 + .../FBuild/FBuildCore/Helpers/Compressor.h | 4 + .../Helpers/ProjectGeneratorBase.cpp | 2 +- .../FBuildCore/Helpers/Report/HTMLReport.h | 2 +- .../FBuildCore/Helpers/Report/JSONReport.cpp | 4 +- .../FBuildCore/Helpers/Report/Report.cpp | 4 +- .../FBuildCore/Helpers/ToolManifest.cpp | 6 +- .../FBuild/FBuildCore/Helpers/ToolManifest.h | 2 +- .../FBuildCore/Helpers/VSProjectGenerator.cpp | 151 +- .../Helpers/XCodeProjectGenerator.cpp | 12 +- .../FBuild/FBuildCore/Protocol/Client.cpp | 10 +- .../FBuild/FBuildCore/Protocol/Server.cpp | 33 +- .../Tools/FBuild/FBuildCore/Protocol/Server.h | 2 +- .../FBuild/FBuildCore/WorkerPool/Job.cpp | 13 +- Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h | 3 +- .../FBuild/FBuildCore/WorkerPool/JobQueue.cpp | 62 +- .../FBuild/FBuildCore/WorkerPool/JobQueue.h | 3 +- .../FBuildCore/WorkerPool/JobQueueRemote.cpp | 9 +- .../FBuildCore/WorkerPool/JobQueueRemote.h | 2 + .../WorkerPool/WorkerBrokerageServer.cpp | 2 +- .../FBuildCore/WorkerPool/WorkerThread.cpp | 16 +- .../FBuildCore/WorkerPool/WorkerThread.h | 5 +- .../RelativePaths/SubDir/subdir.bff | 2 +- .../if_file_exists_directive.bff | 2 +- .../LightCache_IncludeHierarchy/common.h | 2 +- .../LightCache_IncludeHierarchy/fbuild.bff | 2 +- .../LightCache_IncludeUsingMacro/fbuild.bff | 2 +- .../LightCache_IncludeUsingMacro2/fbuild.bff | 2 +- .../LightCache_IncludeUsingMacro2/file.1.cpp | 2 +- .../LightCache_IncludeUsingMacro2/file.2.cpp | 2 +- .../LightCache_IncludeUsingMacro3/fbuild.bff | 2 +- .../RemoteRaceSystemFailure/Fast.cpp | 2 +- .../TestGraph/DatabaseLocation/fbuild.bff | 16 + .../Data/TestObjectList/Exclusions/fbuild.bff | 2 +- .../Data/TestPrecompiledHeaders/fbuild.bff | 4 +- .../Solution_ExternalProject/fbuild.bff | 2 +- .../fbuild_WrongData.bff | 8 +- Code/Tools/FBuild/FBuildTest/FBuildTest.bff | 1 + .../FBuild/FBuildTest/Tests/FBuildTest.h | 2 + .../FBuild/FBuildTest/Tests/TestArgs.cpp | 4 +- .../FBuildTest/Tests/TestBFFParsing.cpp | 35 +- .../FBuild/FBuildTest/Tests/TestCache.cpp | 24 +- .../FBuildTest/Tests/TestCompressor.cpp | 77 +- .../FBuildTest/Tests/TestDependencies.cpp | 2 +- .../Tools/FBuild/FBuildTest/Tests/TestExe.cpp | 20 - .../FBuild/FBuildTest/Tests/TestExec.cpp | 9 +- .../FBuild/FBuildTest/Tests/TestGraph.cpp | 189 +- Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp | 4 +- .../FBuildTest/Tests/TestNodeReflection.cpp | 6 +- .../Tests/TestProjectGeneration.cpp | 2 +- .../FBuild/FBuildTest/Tests/TestRemoveDir.cpp | 16 - .../FBuild/FBuildTest/Tests/TestTest.cpp | 18 - .../FBuild/FBuildTest/Tests/TestUnity.cpp | 2 +- .../FBuild/FBuildWorker/FBuildWorker.bff | 9 +- .../FBuild/FBuildWorker/Worker/Worker.cpp | 8 +- .../FBuildWorker/Worker/WorkerSettings.h | 2 +- .../FBuild/Integration/notepad++markup.xml | 2 +- Code/Tools/FBuild/Integration/usertype.dat | 1 + Code/fbuild.bff | 10 +- External/LZ4/LZ4.bff | 2 +- External/SDK/Clang/Clang.bff | 8 +- External/SDK/Clang/Linux/Clang14.bff | 63 + External/SDK/Clang/Windows/Clang7.bff | 160 - External/SDK/GCC/GCC.bff | 4 + External/SDK/GCC/Linux/GCC11.bff | 66 + External/Zstd/Zstd.bff | 147 + External/Zstd/zstd-1.5.5/CHANGELOG | 800 ++ External/Zstd/zstd-1.5.5/COPYING | 339 + External/Zstd/zstd-1.5.5/LICENSE | 30 + External/Zstd/zstd-1.5.5/README.md | 223 + External/Zstd/zstd-1.5.5/lib/README.md | 224 + .../Zstd/zstd-1.5.5/lib/common/allocations.h | 55 + External/Zstd/zstd-1.5.5/lib/common/bits.h | 200 + .../Zstd/zstd-1.5.5/lib/common/bitstream.h | 437 + .../Zstd/zstd-1.5.5/lib/common/compiler.h | 358 + External/Zstd/zstd-1.5.5/lib/common/cpu.h | 213 + External/Zstd/zstd-1.5.5/lib/common/debug.c | 24 + External/Zstd/zstd-1.5.5/lib/common/debug.h | 107 + .../zstd-1.5.5/lib/common/entropy_common.c | 340 + .../zstd-1.5.5/lib/common/error_private.c | 63 + .../zstd-1.5.5/lib/common/error_private.h | 159 + External/Zstd/zstd-1.5.5/lib/common/fse.h | 639 ++ .../zstd-1.5.5/lib/common/fse_decompress.c | 311 + External/Zstd/zstd-1.5.5/lib/common/huf.h | 273 + External/Zstd/zstd-1.5.5/lib/common/mem.h | 435 + External/Zstd/zstd-1.5.5/lib/common/pool.c | 371 + External/Zstd/zstd-1.5.5/lib/common/pool.h | 90 + .../lib/common/portability_macros.h | 156 + .../Zstd/zstd-1.5.5/lib/common/threading.c | 176 + .../Zstd/zstd-1.5.5/lib/common/threading.h | 150 + External/Zstd/zstd-1.5.5/lib/common/xxhash.c | 24 + External/Zstd/zstd-1.5.5/lib/common/xxhash.h | 5686 +++++++++++++ .../Zstd/zstd-1.5.5/lib/common/zstd_common.c | 48 + .../Zstd/zstd-1.5.5/lib/common/zstd_deps.h | 111 + .../zstd-1.5.5/lib/common/zstd_internal.h | 392 + .../Zstd/zstd-1.5.5/lib/common/zstd_trace.h | 163 + .../Zstd/zstd-1.5.5/lib/compress/clevels.h | 134 + .../zstd-1.5.5/lib/compress/fse_compress.c | 624 ++ External/Zstd/zstd-1.5.5/lib/compress/hist.c | 181 + External/Zstd/zstd-1.5.5/lib/compress/hist.h | 75 + .../zstd-1.5.5/lib/compress/huf_compress.c | 1435 ++++ .../zstd-1.5.5/lib/compress/zstd_compress.c | 7032 +++++++++++++++++ .../lib/compress/zstd_compress_internal.h | 1532 ++++ .../lib/compress/zstd_compress_literals.c | 235 + .../lib/compress/zstd_compress_literals.h | 39 + .../lib/compress/zstd_compress_sequences.c | 442 ++ .../lib/compress/zstd_compress_sequences.h | 54 + .../lib/compress/zstd_compress_superblock.c | 577 ++ .../lib/compress/zstd_compress_superblock.h | 32 + .../Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h | 742 ++ .../lib/compress/zstd_double_fast.c | 758 ++ .../lib/compress/zstd_double_fast.h | 39 + .../Zstd/zstd-1.5.5/lib/compress/zstd_fast.c | 960 +++ .../Zstd/zstd-1.5.5/lib/compress/zstd_fast.h | 38 + .../Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c | 2157 +++++ .../Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h | 127 + .../Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c | 724 ++ .../Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h | 117 + .../lib/compress/zstd_ldm_geartab.h | 106 + .../Zstd/zstd-1.5.5/lib/compress/zstd_opt.c | 1472 ++++ .../Zstd/zstd-1.5.5/lib/compress/zstd_opt.h | 56 + .../zstd-1.5.5/lib/compress/zstdmt_compress.c | 1867 +++++ .../zstd-1.5.5/lib/compress/zstdmt_compress.h | 113 + .../lib/decompress/huf_decompress.c | 1882 +++++ .../lib/decompress/huf_decompress_amd64.S | 576 ++ .../zstd-1.5.5/lib/decompress/zstd_ddict.c | 244 + .../zstd-1.5.5/lib/decompress/zstd_ddict.h | 44 + .../lib/decompress/zstd_decompress.c | 2355 ++++++ .../lib/decompress/zstd_decompress_block.c | 2192 +++++ .../lib/decompress/zstd_decompress_block.h | 73 + .../lib/decompress/zstd_decompress_internal.h | 238 + External/Zstd/zstd-1.5.5/lib/zdict.h | 474 ++ External/Zstd/zstd-1.5.5/lib/zstd.h | 3020 +++++++ External/Zstd/zstd-1.5.5/lib/zstd_errors.h | 114 + 233 files changed, 47657 insertions(+), 1424 deletions(-) create mode 100644 Code/Core/CoreTest/Tests/TestThreadPool.cpp create mode 100644 Code/Core/Process/ThreadPool.cpp create mode 100644 Code/Core/Process/ThreadPool.h create mode 100644 Code/Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff create mode 100644 External/SDK/Clang/Linux/Clang14.bff delete mode 100644 External/SDK/Clang/Windows/Clang7.bff create mode 100644 External/SDK/GCC/Linux/GCC11.bff create mode 100644 External/Zstd/Zstd.bff create mode 100644 External/Zstd/zstd-1.5.5/CHANGELOG create mode 100644 External/Zstd/zstd-1.5.5/COPYING create mode 100644 External/Zstd/zstd-1.5.5/LICENSE create mode 100644 External/Zstd/zstd-1.5.5/README.md create mode 100644 External/Zstd/zstd-1.5.5/lib/README.md create mode 100644 External/Zstd/zstd-1.5.5/lib/common/allocations.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/bits.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/bitstream.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/compiler.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/cpu.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/debug.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/debug.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/entropy_common.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/error_private.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/error_private.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/fse.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/fse_decompress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/huf.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/mem.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/pool.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/pool.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/portability_macros.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/threading.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/threading.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/xxhash.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/xxhash.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/zstd_common.c create mode 100644 External/Zstd/zstd-1.5.5/lib/common/zstd_deps.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/zstd_internal.h create mode 100644 External/Zstd/zstd-1.5.5/lib/common/zstd_trace.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/clevels.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/fse_compress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/hist.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/hist.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/huf_compress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_internal.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm_geartab.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.h create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.h create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress_amd64.S create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.c create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.h create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress.c create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.c create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.h create mode 100644 External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_internal.h create mode 100644 External/Zstd/zstd-1.5.5/lib/zdict.h create mode 100644 External/Zstd/zstd-1.5.5/lib/zstd.h create mode 100644 External/Zstd/zstd-1.5.5/lib/zstd_errors.h diff --git a/Code/.FASTBuild/HelperFunctions.bff b/Code/.FASTBuild/HelperFunctions.bff index 62102fd1d..90d6e31df 100644 --- a/Code/.FASTBuild/HelperFunctions.bff +++ b/Code/.FASTBuild/HelperFunctions.bff @@ -189,5 +189,5 @@ function CreateVCXProject_Exe( .Name Alias( '$ProjectName$-OSX-$BuildConfigName$' ) { .Targets = '$ProjectName$-Exe-OSX-$BuildConfigName$' } } #endif - + //------------------------------------------------------------------------------ diff --git a/Code/Core/CoreTest/TestMain.cpp b/Code/Core/CoreTest/TestMain.cpp index 670a32d01..21062fc1a 100644 --- a/Code/Core/CoreTest/TestMain.cpp +++ b/Code/Core/CoreTest/TestMain.cpp @@ -29,6 +29,7 @@ int main( int, char *[] ) REGISTER_TESTGROUP( TestSystemMutex ) REGISTER_TESTGROUP( TestTestTCPConnectionPool ) REGISTER_TESTGROUP( TestThread ) + REGISTER_TESTGROUP( TestThreadPool ) REGISTER_TESTGROUP( TestTimer ) REGISTER_TESTGROUP( TestUnorderedMap ) diff --git a/Code/Core/CoreTest/Tests/TestFileIO.cpp b/Code/Core/CoreTest/Tests/TestFileIO.cpp index f56181451..0f3ff37bb 100644 --- a/Code/Core/CoreTest/Tests/TestFileIO.cpp +++ b/Code/Core/CoreTest/Tests/TestFileIO.cpp @@ -320,7 +320,7 @@ void TestFileIO::LongPaths() const // // Ensure long paths are correctly handled by various functions // - + #if defined( __WINDOWS__ ) // On Windows, long path support must be enabled via a registry key // Only if this is enabled can we expect our test to pass @@ -365,7 +365,7 @@ void TestFileIO::LongPaths() const filePathB.Trim( 0, 5 ); filePathB += ".copy"; TEST_ASSERT( filePathB.GetLength() == ( filePathA.GetLength() ) ); - + // long subdir 1 subDir1.Format( "%s/%s", tmpPath2.Get(), a.Get() ); TEST_ASSERT( subDir1.GetLength() == ( tmpPath2.GetLength() + 1 + 255 ) ); diff --git a/Code/Core/CoreTest/Tests/TestSemaphore.cpp b/Code/Core/CoreTest/Tests/TestSemaphore.cpp index c270ab50a..f93c09dd0 100644 --- a/Code/Core/CoreTest/Tests/TestSemaphore.cpp +++ b/Code/Core/CoreTest/Tests/TestSemaphore.cpp @@ -105,7 +105,7 @@ void TestSemaphore::WaitTimeout() const void TestSemaphore::MaxCount() const { // Only Windows supports a signall count limit for Semaphores - + // Create sempahore with a max count Semaphore s( 1 ); diff --git a/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp b/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp index 819af83fb..bb110bd94 100644 --- a/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp +++ b/Code/Core/CoreTest/Tests/TestSmallBlockAllocator.cpp @@ -39,7 +39,7 @@ class TestSmallBlockAllocator : public TestGroup // Helper functions static void GetRandomAllocSizes( const uint32_t numAllocs, Array< uint32_t> & allocSizes ); static float AllocateFromSystemAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount ); - static float AllocateFromSmallBlockAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount, const bool threadSafe = true ); + static float AllocateFromSmallBlockAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount ); static uint32_t ThreadFunction_System( void * userData ); static uint32_t ThreadFunction_SmallBlock( void * userData ); }; @@ -67,12 +67,10 @@ void TestSmallBlockAllocator::SingleThreaded() const const float time1 = AllocateFromSystemAllocator( allocSizes, repeatCount ); const float time2 = AllocateFromSmallBlockAllocator( allocSizes, repeatCount ); - const float time3 = AllocateFromSmallBlockAllocator( allocSizes, repeatCount, false ); // Thread-safe = false // output - OUTPUT( "System (malloc) : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time1, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time1 ) ); - OUTPUT( "SmallBlockAllocator : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time2, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time2 ) ); - OUTPUT( "SmallBlockAllocator (Single-Threaded mode) : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time3, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time3 ) ); + OUTPUT( "System (malloc) : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time1, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time1 ) ); + OUTPUT( "SmallBlockAllocator : %2.3fs - %u allocs @ %u allocs/sec\n", (double)time2, ( numAllocs * repeatCount ), (uint32_t)( float( numAllocs * repeatCount ) / time2 ) ); } // MultiThreaded @@ -191,18 +189,13 @@ void TestSmallBlockAllocator::MultiThreaded() const // AllocateFromSmallBlockAllocator //------------------------------------------------------------------------------ -/*static*/ float TestSmallBlockAllocator::AllocateFromSmallBlockAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount, const bool threadSafe ) +/*static*/ float TestSmallBlockAllocator::AllocateFromSmallBlockAllocator( const Array< uint32_t > & allocSizes, const uint32_t repeatCount ) { const size_t numAllocs = allocSizes.GetSize(); Array< void * > allocs( numAllocs, false ); const Timer timer; - if ( threadSafe == false ) - { - SmallBlockAllocator::SetSingleThreadedMode( true ); - } - for ( size_t r = 0; r < repeatCount; ++r ) { // Use ALLOC @@ -222,11 +215,6 @@ void TestSmallBlockAllocator::MultiThreaded() const allocs.Clear(); } - if ( threadSafe == false ) - { - SmallBlockAllocator::SetSingleThreadedMode( false ); - } - return timer.GetElapsed(); } diff --git a/Code/Core/CoreTest/Tests/TestThreadPool.cpp b/Code/Core/CoreTest/Tests/TestThreadPool.cpp new file mode 100644 index 000000000..0e72ae6f3 --- /dev/null +++ b/Code/Core/CoreTest/Tests/TestThreadPool.cpp @@ -0,0 +1,89 @@ +// TestThreadPool.cpp +//------------------------------------------------------------------------------ + +// Includes +//------------------------------------------------------------------------------ +// TestFramework +#include "TestFramework/TestGroup.h" + +// Core +#include "Core/Process/Atomic.h" +#include "Core/Process/ThreadPool.h" + +// TestThread +//------------------------------------------------------------------------------ +class TestThreadPool : public TestGroup +{ +private: + DECLARE_TESTS + + // Tests + void Unused() const; + void SingleJob() const; + void MultipleJobs() const; + + // Helpers + static void Increment( void * userData ) + { + Atomic * u32 = static_cast *>( userData ); + u32->Increment(); + } +}; + +// Register Tests +//------------------------------------------------------------------------------ +REGISTER_TESTS_BEGIN( TestThreadPool ) + REGISTER_TEST( Unused ) + REGISTER_TEST( SingleJob ) + REGISTER_TEST( MultipleJobs ) +REGISTER_TESTS_END + +// Unused +//------------------------------------------------------------------------------ +void TestThreadPool::Unused() const +{ + // A thread pool never used + ThreadPool threadPool( 4 ); +} + +// SingleJob +//------------------------------------------------------------------------------ +void TestThreadPool::SingleJob() const +{ + Atomic count; + + // Create + ThreadPool * threadPool = FNEW( ThreadPool( 4 ) ); + + // Enqueue jobs + threadPool->EnqueueJob( Increment, &count ); + + // Destroy + FDELETE threadPool; + + TEST_ASSERT( count.Load() == 1 ); +} + +// MultipleJobs +//------------------------------------------------------------------------------ +void TestThreadPool::MultipleJobs() const +{ + Atomic count; + + // Create + ThreadPool * threadPool = FNEW( ThreadPool( 4 ) ); + + // Enqueue jobs + const uint32_t numJobs = 1024; + for ( uint32_t i = 0; i < numJobs; ++i ) + { + threadPool->EnqueueJob( Increment, &count ); + } + + // Destroy + FDELETE threadPool; + + TEST_ASSERT( count.Load() == numJobs ); +} + +//------------------------------------------------------------------------------ diff --git a/Code/Core/Env/Assert.cpp b/Code/Core/Env/Assert.cpp index e7cc80dec..fe99ec5cc 100644 --- a/Code/Core/Env/Assert.cpp +++ b/Code/Core/Env/Assert.cpp @@ -52,7 +52,7 @@ bool IsDebuggerAttached() // reason, we get a predictable result. struct kinfo_proc info; info.kp_proc.p_flag = 0; - + // Initialize mib, which tells sysctl the info we want, in this case // we're looking for information about a specific process ID. int mib[ 4 ]; @@ -60,11 +60,11 @@ bool IsDebuggerAttached() mib[ 1 ] = KERN_PROC; mib[ 2 ] = KERN_PROC_PID; mib[ 3 ] = getpid(); - + // Call sysctl size_t size = sizeof(info); VERIFY( sysctl( mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0 ) == 0 ); - + // We're being debugged if the P_TRACED flag is set. return ( ( info.kp_proc.p_flag & P_TRACED ) != 0 ); #elif defined( __LINUX__ ) diff --git a/Code/Core/Env/Env.cpp b/Code/Core/Env/Env.cpp index 275ca565b..36a6955de 100644 --- a/Code/Core/Env/Env.cpp +++ b/Code/Core/Env/Env.cpp @@ -223,7 +223,7 @@ void Env::GetExePath( AString & output ) uint32_t bufferSize = 0; VERIFY( _NSGetExecutablePath( nullptr, &bufferSize ) == -1 ); ASSERT( bufferSize > 0 ); // Updated by _NSGetExecutablePath - + // Reserve enough space (-1 since bufferSize includes the null) output.SetLength( bufferSize - 1 ); VERIFY( _NSGetExecutablePath( output.Get(), &bufferSize ) == 0 ); diff --git a/Code/Core/Env/Types.h b/Code/Core/Env/Types.h index 09775fa22..61e9c8e31 100644 --- a/Code/Core/Env/Types.h +++ b/Code/Core/Env/Types.h @@ -88,13 +88,6 @@ typedef signed int int32_t; #endif #endif -// Versions of Visual Studio prior to 2017 don't manage noexcept properly -#if defined( _MSC_VER ) && ( _MSC_VER < 1910 ) && !defined( __clang__ ) - #define NOEXCEPT -#else - #define NOEXCEPT noexcept -#endif - #ifndef LONGLONG typedef long long LONGLONG; #endif diff --git a/Code/Core/Env/WindowsHeader.h b/Code/Core/Env/WindowsHeader.h index 2f0a9727d..11cf8dfa8 100644 --- a/Code/Core/Env/WindowsHeader.h +++ b/Code/Core/Env/WindowsHeader.h @@ -1,7 +1,7 @@ // WindowsHeader //------------------------------------------------------------------------------ // -// Windows.h and WinSock2.h mustbe included in a specific order to avoid +// Windows.h and WinSock2.h must be included in a specific order to avoid // compile errors. This is problematic when headers include other headers // that include Windows.h, or when using Unity. // @@ -16,7 +16,7 @@ // Includes //------------------------------------------------------------------------------ -#pragma warning(push, 0) +#pragma warning(push, 0) #pragma warning(push) #pragma warning(disable:6101) // Returning uninitialized memory '*Mtu'. A successful path through the function does not set the named _Out_ parameter. #include // WinSock2.h must be first diff --git a/Code/Core/FileIO/FileIO.cpp b/Code/Core/FileIO/FileIO.cpp index 8cb5615d7..ad29381d1 100644 --- a/Code/Core/FileIO/FileIO.cpp +++ b/Code/Core/FileIO/FileIO.cpp @@ -58,13 +58,13 @@ { public: typedef int (*FuncPtr)( int dirfd, const char * pathname, const struct timespec times[ 2 ], int flags ); - + OSXHelper_utimensat() { // Open the c runtime library m_LibCHandle = dlopen( "libc.dylib", RTLD_LAZY ); ASSERT( m_LibCHandle ); // This should never fail - + // See if utimensat exists m_FuncPtr = (FuncPtr)dlsym( m_LibCHandle, "utimensat" ); } @@ -260,6 +260,8 @@ ssize_t bytesCopied = 0; ssize_t offset = 0; + bool sendfileUnavailable = false; + while ( offset < stat_source.st_size ) { // sendfile has an arbitrary limit of 0x7ffff000 on all systems (even if 64bit) @@ -269,11 +271,46 @@ const ssize_t sent = sendfile( dest, source, &offset, count ); if ( sent <= 0 ) { + // sendfile manual suggests defaulting to read/write functions + if ( ( sent == -1 ) && ( ( errno == EINVAL ) || ( errno == ENOSYS ) ) ) + { + sendfileUnavailable = true; + } + break; // Copy failed (incomplete) } bytesCopied += sent; } + // manually copy source file to destination in fixed-size chunks + // continues until source EOF is reached or any data fails to copy + if ( sendfileUnavailable ) + { + const size_t count = 4096; + void * buf = ALLOC( count ); + + while ( bytesCopied < stat_source.st_size ) + { + const ssize_t readBytes = read( source, buf, count ); + + if ( readBytes <= 0 ) + { + break; + } + + const ssize_t written = write( dest, buf, readBytes ); + + if ( written != readBytes ) + { + break; + } + + bytesCopied += written; + } + + FREE( buf ); + } + close( source ); close( dest ); @@ -321,6 +358,17 @@ return ( results->GetSize() != oldSize ); } +// GetFiles +//------------------------------------------------------------------------------ +/*static*/ void FileIO::GetFiles( const AString & path, + GetFilesHelper & helper ) +{ + // make a copy of the path as it will be modified during recursion + AStackString<> pathCopy( path ); + PathUtils::EnsureTrailingSlash( pathCopy ); + GetFilesRecurse( pathCopy, helper ); +} + // GetFilesEx //------------------------------------------------------------------------------ /*static*/ bool FileIO::GetFilesEx( const AString & path, @@ -677,7 +725,7 @@ } continue; // Keep processing path } - + // Path doesn't exist so keep rest of path as-is outNormalizedPath += pos; break; @@ -812,7 +860,7 @@ t[ 1 ] = t[ 0 ]; return ( (gOSXHelper_utimensat.m_FuncPtr)( 0, fileName.Get(), t, 0 ) == 0 ); } - + // Fallback to regular low-resolution filetime setting struct timeval t[ 2 ]; t[ 0 ].tv_sec = fileTime / 1000000000ULL; @@ -843,7 +891,7 @@ { return ( (gOSXHelper_utimensat.m_FuncPtr)( 0, fileName.Get(), nullptr, 0 ) == 0 ); } - + // Fallback to regular low-resolution filetime setting return ( utimes( fileName.Get(), nullptr ) == 0 ); #elif defined( __LINUX__ ) @@ -964,6 +1012,150 @@ } #endif +//------------------------------------------------------------------------------ +/*static*/ void FileIO::GetFilesRecurse( AString & pathCopy, GetFilesHelper & helper ) +{ + const uint32_t baseLength = pathCopy.GetLength(); + + // Windows requires the path contain a wildcard + #if defined( __WINDOWS__ ) + pathCopy += '*'; // retrieve all files/folders + #endif + + // Don't traverse into symlinks - TODO:B Why is this OSX/Linux only? + #if defined( __LINUX__ ) || defined( __APPLE__ ) + // Special case symlinks + struct stat stat_source; + if ( lstat( pathCopy.Get(), &stat_source ) != 0 ) + { + return; + } + if ( S_ISLNK( stat_source.st_mode ) ) + { + return; + } + #endif + + // Start dir list operation + #if defined( __WINDOWS__ ) + WIN32_FIND_DATA findData; + HANDLE hFind = FindFirstFileEx( pathCopy.Get(), FindExInfoBasic, &findData, FindExSearchNameMatch, nullptr, 0 ); + if ( hFind == INVALID_HANDLE_VALUE) + { + return; + } + #else + DIR * dir = opendir( pathCopy.Get() ); + if ( dir == nullptr ) + { + return; + } + #endif + + // Iterate entries + #if defined( __LINUX__ ) || defined( __APPLE__ ) + for ( ;; ) + #else + do + #endif + { + #if defined( __LINUX__ ) || defined( __APPLE__ ) + dirent * entry = readdir( dir ); + if ( entry == nullptr ) + { + break; // no more entries + } + #endif + + // Name of this entry + #if defined( __WINDOWS__ ) + const char * const entryName = findData.cFileName; + #else + const char * const entryName = entry->d_name; + #endif + + // Determine if entry is a directory + #if defined( __WINDOWS__ ) + const bool isDir = ( ( findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) == FILE_ATTRIBUTE_DIRECTORY ); + #else + bool isDir = ( entry->d_type == DT_DIR ); + + // Not all filesystems have support for returning the file type in + // d_type and applications must properly handle a return of DT_UNKNOWN. + if ( entry->d_type == DT_UNKNOWN ) + { + pathCopy.SetLength( baseLength ); + pathCopy += entry->d_name; + + struct stat info; + VERIFY( lstat( pathCopy.Get(), &info ) == 0 ); + isDir = S_ISDIR( info.st_mode ); + } + #endif + + // Directory? + if ( isDir ) + { + // ignore magic '.' and '..' folders + // (don't need to check length of name, as all names are at least 1 char + // which means index 0 and 1 are valid to access) + if ( ( entryName[ 0 ] == '.' ) && + ( ( entryName[ 1 ] == '.' ) || ( entryName[ 1 ] == 0 ) ) ) + { + continue; + } + + // Process Dir + pathCopy.SetLength( baseLength ); + pathCopy += entryName; + pathCopy += NATIVE_SLASH; + const bool recurseIntoDir = helper.OnDirectory( pathCopy ); + if ( recurseIntoDir ) + { + GetFilesRecurse( pathCopy, helper ); + } + continue; + } + + // Process File + pathCopy.SetLength( baseLength ); + if ( helper.ShouldIncludeFile( entryName ) ) + { + FileInfo fileInfo; + const uint32_t fileNameLen = static_cast( AString::StrLen( entryName ) ); + const uint32_t fullPathLen = pathCopy.GetLength() + fileNameLen; + fileInfo.m_Name.SetReserved( fullPathLen ); + fileInfo.m_Name.Assign( pathCopy ); + fileInfo.m_Name.Append(entryName, fileNameLen ); + #if defined( __WINDOWS__ ) + fileInfo.m_Attributes = findData.dwFileAttributes; + fileInfo.m_LastWriteTime = (uint64_t)findData.ftLastWriteTime.dwLowDateTime | ( (uint64_t)findData.ftLastWriteTime.dwHighDateTime << 32 ); + fileInfo.m_Size = (uint64_t)findData.nFileSizeLow | ( (uint64_t)findData.nFileSizeHigh << 32 ); + #else + struct stat info; + VERIFY( lstat( fileInfo.m_Name.Get(), &info ) == 0 ); + fileInfo.m_Attributes = info.st_mode; + #if defined( __APPLE__ ) + fileInfo.m_LastWriteTime = ( ( (uint64_t)info.st_mtimespec.tv_sec * 1000000000ULL ) + (uint64_t)info.st_mtimespec.tv_nsec ); + #else + fileInfo.m_LastWriteTime = ( ( (uint64_t)info.st_mtim.tv_sec * 1000000000ULL ) + (uint64_t)info.st_mtim.tv_nsec ); + #endif + fileInfo.m_Size = info.st_size; + #endif + helper.OnFile( Move( fileInfo ) ); + } + } + #if defined( __WINDOWS__ ) + while ( FindNextFile( hFind, &findData ) != 0 ); + #endif + + #if defined( __WINDOWS__ ) + FindClose( hFind ); + #else + closedir( dir ); + #endif +} + // GetFilesRecurse //------------------------------------------------------------------------------ /*static*/ void FileIO::GetFilesRecurse( AString & pathCopy, @@ -1608,4 +1800,44 @@ bool FileIO::FileInfo::IsReadOnly() const return false; } +// GetFilesHelper CONSTRUCTOR +//------------------------------------------------------------------------------ +GetFilesHelper::GetFilesHelper( size_t sizeHint ) + : m_Files( sizeHint, true ) +{ +} + +//------------------------------------------------------------------------------ +GetFilesHelper::GetFilesHelper( const Array & patterns, size_t sizeHint ) + : m_Patterns( &patterns ) + , m_Files( sizeHint, true ) +{ +} + +// GetFilesHelper DESTRUCTOR +//------------------------------------------------------------------------------ +GetFilesHelper::~GetFilesHelper() = default; + +// OnDirectory +//------------------------------------------------------------------------------ +/*virtual*/ bool GetFilesHelper::OnDirectory( const AString & /*dirPath*/ ) +{ + return m_Recurse; +} + +// ShouldIncludeFile +//------------------------------------------------------------------------------ +/*virtual*/ bool GetFilesHelper::ShouldIncludeFile( const char * fileName ) +{ + // Check wildcard patterns + return FileIO::IsMatch( m_Patterns, fileName ); +} + +// OnFile +//------------------------------------------------------------------------------ +/*virtual*/ void GetFilesHelper::OnFile( FileIO::FileInfo && fileInfo ) +{ + m_Files.EmplaceBack( Move( fileInfo ) ); +} + //------------------------------------------------------------------------------ diff --git a/Code/Core/FileIO/FileIO.h b/Code/Core/FileIO/FileIO.h index 276de7d23..0e33a44df 100644 --- a/Code/Core/FileIO/FileIO.h +++ b/Code/Core/FileIO/FileIO.h @@ -12,6 +12,10 @@ //------------------------------------------------------------------------------ #define MAX_PATH 260 +// Forward Declarations +//------------------------------------------------------------------------------ +class GetFilesHelper; + // FileIO //------------------------------------------------------------------------------ class FileIO @@ -28,6 +32,8 @@ class FileIO const AString & wildCard, bool recurse, Array< AString > * results ); + static void GetFiles( const AString & path, + GetFilesHelper & helper ); struct FileInfo { AString m_Name; @@ -87,6 +93,8 @@ class FileIO static bool IsWindowsLongPathSupportEnabledInternal(); #endif + static void GetFilesRecurse( AString & path, + GetFilesHelper & helper ); static void GetFilesRecurse( AString & path, const AString & wildCard, Array< AString > * results ); @@ -99,7 +107,32 @@ class FileIO static void GetFilesNoRecurseEx( const char * path, const Array< AString > * patterns, Array< FileInfo > * results ); + + friend class GetFilesHelper; static bool IsMatch( const Array< AString > * patterns, const char * fileName ); }; //------------------------------------------------------------------------------ +class GetFilesHelper +{ +public: + explicit GetFilesHelper( size_t sizeHint = 1024 ); + explicit GetFilesHelper( const Array & patterns, size_t sizeHint = 1024 ); + virtual ~GetFilesHelper(); + + // Default implementation can be extended or customized + [[nodiscard]] virtual bool OnDirectory( const AString & dirPath ); + [[nodiscard]] virtual bool ShouldIncludeFile( const char * fileName ); + virtual void OnFile( FileIO::FileInfo && fileInfo ); + + // Access results + const Array & GetFiles() const { return m_Files; } + Array & GetFiles() { return m_Files; } + +protected: + bool m_Recurse = true; + const Array * m_Patterns = nullptr; + Array m_Files; +}; + +//------------------------------------------------------------------------------ diff --git a/Code/Core/Mem/Mem.cpp b/Code/Core/Mem/Mem.cpp index d347d5903..bb901156a 100644 --- a/Code/Core/Mem/Mem.cpp +++ b/Code/Core/Mem/Mem.cpp @@ -139,14 +139,14 @@ void Free( void * ptr ) // Directly called void * operator new( size_t size ) { return AllocFileLine( size, "Unknown", 0 ); } void * operator new[]( size_t size ) { return AllocFileLine( size, "Unknown", 0 ); } - void operator delete( void * ptr ) NOEXCEPT { Free( ptr ); } - void operator delete[]( void * ptr ) NOEXCEPT { Free( ptr ); } + void operator delete( void * ptr ) noexcept { Free( ptr ); } + void operator delete[]( void * ptr ) noexcept { Free( ptr ); } #else #if !__has_feature( address_sanitizer ) && !__has_feature( memory_sanitizer ) && !__has_feature( thread_sanitizer ) && !defined( __SANITIZE_ADDRESS__ ) void * operator new( size_t size ) { return Alloc( size ); } void * operator new[]( size_t size ) { return Alloc( size ); } - void operator delete( void * ptr ) NOEXCEPT { Free( ptr ); } - void operator delete[]( void * ptr ) NOEXCEPT { Free( ptr ); } + void operator delete( void * ptr ) noexcept { Free( ptr ); } + void operator delete[]( void * ptr ) noexcept { Free( ptr ); } #endif #endif //------------------------------------------------------------------------------ diff --git a/Code/Core/Mem/Mem.h b/Code/Core/Mem/Mem.h index 1c3bd692a..abef3858e 100644 --- a/Code/Core/Mem/Mem.h +++ b/Code/Core/Mem/Mem.h @@ -61,8 +61,8 @@ void Free( void * ptr ); #if !__has_feature( address_sanitizer ) && !__has_feature( memory_sanitizer ) && !__has_feature( thread_sanitizer ) && !defined( __SANITIZE_ADDRESS__ ) void * operator new( size_t size ); void * operator new[]( size_t size ); -void operator delete( void * ptr ) NOEXCEPT; -void operator delete[]( void * ptr ) NOEXCEPT; +void operator delete( void * ptr ) noexcept; +void operator delete[]( void * ptr ) noexcept; #endif //------------------------------------------------------------------------------ diff --git a/Code/Core/Mem/SmallBlockAllocator.cpp b/Code/Core/Mem/SmallBlockAllocator.cpp index d56fb85f7..d03502055 100644 --- a/Code/Core/Mem/SmallBlockAllocator.cpp +++ b/Code/Core/Mem/SmallBlockAllocator.cpp @@ -32,10 +32,6 @@ // Static Data //------------------------------------------------------------------------------ -/*static*/ bool SmallBlockAllocator::s_ThreadSafeAllocs( true ); -#if defined( ASSERTS_ENABLED ) - /*static*/ uint64_t SmallBlockAllocator::s_ThreadSafeAllocsDebugOwnerThread( 0 ); -#endif /*static*/ void * SmallBlockAllocator::s_BucketMemoryStart( MEM_BUCKETS_NOT_INITIALIZED ); /*static*/ uint32_t SmallBlockAllocator::s_BucketNextFreePageIndex( 0 ); /*static*/ uint64_t SmallBlockAllocator::s_BucketMemBucketMemory[ BUCKET_NUM_BUCKETS * sizeof( MemBucket ) / sizeof (uint64_t) ]; @@ -143,19 +139,11 @@ void * SmallBlockAllocator::Alloc( size_t size, size_t align ) return nullptr; // Can't satify alignment } - // Sanity check that we're being used safely - ASSERT( s_ThreadSafeAllocs || ( s_ThreadSafeAllocsDebugOwnerThread == (uint64_t)Thread::GetCurrentThreadId() ) ); - - void* ptr; + void* ptr; // Alloc - if ( s_ThreadSafeAllocs ) - { - MutexHolder mh(bucket.m_Mutex); - ptr = bucket.Alloc(); - } - else { + MutexHolder mh( bucket.m_Mutex ); ptr = bucket.Alloc(); } @@ -193,55 +181,15 @@ bool SmallBlockAllocator::Free( void * ptr ) MemDebug::FillMem( ptr, bucket.m_BlockSize, MemDebug::MEM_FILL_FREED_ALLOCATION_PATTERN ); #endif - // Sanity check that we're being used safely - ASSERT( s_ThreadSafeAllocs || ( s_ThreadSafeAllocsDebugOwnerThread == (uint64_t)Thread::GetCurrentThreadId() ) ); - // Free it - if ( s_ThreadSafeAllocs ) - { - bucket.m_Mutex.Lock(); - } - - bucket.Free( ptr ); - - if ( s_ThreadSafeAllocs ) { - bucket.m_Mutex.Unlock(); + MutexHolder mh( bucket.m_Mutex ); + bucket.Free( ptr ); } return true; } -// SetSingleThreadedMode -//------------------------------------------------------------------------------ -/*static*/ void SmallBlockAllocator::SetSingleThreadedMode( bool singleThreadedMode ) -{ - if ( singleThreadedMode ) - { - // Sanity check we're not already in single threaded mode - ASSERT( s_ThreadSafeAllocs == true ); - ASSERT( s_ThreadSafeAllocsDebugOwnerThread == 0 ); - - // Store the new owner thread for further safety checks - #if defined( ASSERTS_ENABLED ) - s_ThreadSafeAllocsDebugOwnerThread = (uint64_t)Thread::GetCurrentThreadId(); - #endif - } - else - { - // Sanity check we're in single threaded mode - ASSERT( s_ThreadSafeAllocs == false ); - ASSERT( s_ThreadSafeAllocsDebugOwnerThread == (uint64_t)Thread::GetCurrentThreadId() ); - - // Store the new owner thread for further safety checks - #if defined( ASSERTS_ENABLED ) - s_ThreadSafeAllocsDebugOwnerThread = 0; - #endif - } - - s_ThreadSafeAllocs = ( !singleThreadedMode ); -} - // AllocateMemoryForPage //------------------------------------------------------------------------------ /*virtual*/ void * SmallBlockAllocator::MemBucket::AllocateMemoryForPage() diff --git a/Code/Core/Mem/SmallBlockAllocator.h b/Code/Core/Mem/SmallBlockAllocator.h index 4364ef6f0..bcc49303f 100644 --- a/Code/Core/Mem/SmallBlockAllocator.h +++ b/Code/Core/Mem/SmallBlockAllocator.h @@ -19,7 +19,6 @@ // Includes //------------------------------------------------------------------------------ -#include "Core/Env/Assert.h" #include "Core/Mem/MemPoolBlock.h" #include "Core/Process/Mutex.h" @@ -34,9 +33,6 @@ // Attempt to free. Returns false if not a bucket owned allocation static bool Free( void * ptr ); - // Hint when operating only on a single thread as we can greatly reduce allocation cost - static void SetSingleThreadedMode( bool singleThreadedMode ); - #if defined( DEBUG ) static void DumpStats(); #endif @@ -69,15 +65,6 @@ Mutex m_Mutex; }; - friend class MemBucket; - - // Single Threaded Mode - static bool s_ThreadSafeAllocs; - #if defined( ASSERTS_ENABLED ) - // When in single-threaded mode, catch unsafe use - static uint64_t s_ThreadSafeAllocsDebugOwnerThread; - #endif - // Address space used by allocators static void * s_BucketMemoryStart; static uint32_t s_BucketNextFreePageIndex; // Next free memory page to commit diff --git a/Code/Core/Network/Network.cpp b/Code/Core/Network/Network.cpp index 87913b648..5156455ce 100644 --- a/Code/Core/Network/Network.cpp +++ b/Code/Core/Network/Network.cpp @@ -133,7 +133,7 @@ NI_NUMERICHOST ) == 0 ); outAddresses.EmplaceBack( host ); } - + info = info->ifa_next; } diff --git a/Code/Core/Network/TCPConnectionPool.h b/Code/Core/Network/TCPConnectionPool.h index 97d99752d..a85176386 100644 --- a/Code/Core/Network/TCPConnectionPool.h +++ b/Code/Core/Network/TCPConnectionPool.h @@ -26,7 +26,7 @@ class TCPConnectionPool; // Constants //------------------------------------------------------------------------------ -namespace +namespace { static const uint32_t kDefaultConnectionTimeoutMS = ( 2 * 1000 ); static const uint32_t kDefaultSendTimeoutMS = ( 10 * 60 * 1000 ); diff --git a/Code/Core/Process/Process.cpp b/Code/Core/Process/Process.cpp index 4f315802e..d843970af 100644 --- a/Code/Core/Process/Process.cpp +++ b/Code/Core/Process/Process.cpp @@ -601,7 +601,7 @@ bool Process::ReadAllData( AString & outMem, if ( ( prevOutSize != outMem.GetLength() ) || ( prevErrSize != errMem.GetLength() ) ) { #if defined( __LINUX__ ) - // Reset sleep interval + // Reset sleep interval sleepIntervalMS = 1; #endif continue; // try reading again right away incase there is more @@ -702,7 +702,7 @@ bool Process::ReadAllData( AString & outMem, { ASSERT( false ); // error! } - + // Update length buffer.SetLength( sizeSoFar + bytesReadNow ); } @@ -749,7 +749,7 @@ bool Process::ReadAllData( AString & outMem, ASSERT( false ); // error! result = 0; // no bytes read } - + // Update length buffer.SetLength( buffer.GetLength() + (uint32_t)result ); } diff --git a/Code/Core/Process/Thread.cpp b/Code/Core/Process/Thread.cpp index 76af89591..d014bf7e2 100644 --- a/Code/Core/Process/Thread.cpp +++ b/Code/Core/Process/Thread.cpp @@ -48,12 +48,16 @@ struct ThreadStartInfo ThreadStartInfo( Thread::ThreadEntryFunction entryFunc, void * userData, const char * threadName ) : m_UserEntryFunction( entryFunc ) , m_UserData( userData ) - , m_ThreadName( threadName ) - {} + { + if ( threadName ) + { + m_ThreadName = threadName; + } + } Thread::ThreadEntryFunction m_UserEntryFunction; void * m_UserData; - const char * m_ThreadName; + AString m_ThreadName; #if defined( __WINDOWS__ ) static uint32_t WINAPI ThreadStartFunction( void * userData ) @@ -66,13 +70,16 @@ struct ThreadStartInfo Thread::ThreadEntryFunction realFunction = originalInfo->m_UserEntryFunction; void * realUserData = originalInfo->m_UserData; - // Set thread name for debugging - #ifdef DEBUG - if ( originalInfo->m_ThreadName ) - { - Thread::SetThreadName( originalInfo->m_ThreadName ); - } - #endif + if ( originalInfo->m_ThreadName.IsEmpty() == false ) + { + // Set thread name for debugging + #ifdef DEBUG + Thread::SetThreadName( originalInfo->m_ThreadName.Get() ); + #endif + + // Set thread name for profiling + PROFILE_SET_THREAD_NAME( originalInfo->m_ThreadName.Get() ); + } // done with ThreadStartInfo MemoryBarrier(); @@ -85,6 +92,8 @@ struct ThreadStartInfo return (void *)(size_t)( (*realFunction)( realUserData ) ); #endif } + + void operator =(const ThreadStartInfo& other) = delete; }; // Static Data @@ -112,7 +121,7 @@ void Thread::Start( ThreadEntryFunction func, { // Can only start if not already started ASSERT( !IsRunning() ); - + // Create structure to pass to thread ThreadStartInfo & info = *FNEW( ThreadStartInfo( func, userData, threadName ) ); MemoryBarrier(); diff --git a/Code/Core/Process/ThreadPool.cpp b/Code/Core/Process/ThreadPool.cpp new file mode 100644 index 000000000..907aaa189 --- /dev/null +++ b/Code/Core/Process/ThreadPool.cpp @@ -0,0 +1,173 @@ +// ThreadPool +//------------------------------------------------------------------------------ + +// Includes +//------------------------------------------------------------------------------ +#include "ThreadPool.h" + +// Core +#include "Core/Process/Thread.h" +#include "Core/Profile/Profile.h" +#include "Core/Strings/AStackString.h" + +// CONSTRUCTOR +//------------------------------------------------------------------------------ +ThreadPool::ThreadPool( uint32_t numThreads ) + : m_NumThreads( numThreads ) + #if defined( __WINDOWS__ ) + , m_WakeSemaphore( numThreads ) // On Windows, take advantage of signalling limit + #else + , m_WakeSemaphore() + #endif +{ + PROFILE_FUNCTION; + + // Doesn't make sense to create a pool with no threads + ASSERT( m_NumThreads > 0 ); + + // Start first thread (it will create additional threads) + const uint32_t threadId = 1; + m_FirstThread = FNEW( ThreadPoolThread( threadId, *this ) ); +} + +// DESTRUCTOR +//------------------------------------------------------------------------------ +ThreadPool::~ThreadPool() +{ + PROFILE_FUNCTION; + + // Signal worker threads to exit + m_WantToQuit.Store( true ); + m_WakeSemaphore.Signal( m_NumThreads ); + + // Destroy first thread (it will destroy additional threads) + FDELETE m_FirstThread; + + // All jobs should have been completed + ASSERT( m_JobQueue.IsEmpty() ); +} + +// EnqueueJob +//------------------------------------------------------------------------------ +void ThreadPool::EnqueueJob( ThreadJobFunc func, void * userData ) +{ + // Add job to queue + { + MutexHolder mh( m_JobQueueMutex ); + ThreadJob & job = m_JobQueue.EmplaceBack(); + job.m_Function = func; + job.m_UserData = userData; + } + + // Wake threads if sleeping + m_WakeSemaphore.Signal(); +} + +// ThreadFuncWrapper +//------------------------------------------------------------------------------ +/*static*/ uint32_t ThreadPool::ThreadFuncWrapper( void * userData ) +{ + ThreadPoolThread * threadPoolThread = static_cast( userData ); + threadPoolThread->ThreadFunc(); + return 0; +} + +// ThreadFunc +//------------------------------------------------------------------------------ +void ThreadPool::ThreadFunc() +{ + for ( ;; ) + { + // See if a job is available + // (Jobs might be taken by other threads even if we were woken) + ThreadJob job; + { + MutexHolder mh( m_JobQueueMutex ); + if ( m_JobQueue.IsEmpty() == false ) + { + // Remove Job from queue, processing them in order of enqueuing + // Popping from the front of an Array is inefficient, but current + // uses have few jobs (typically one per-core) + job = m_JobQueue[ 0 ]; + m_JobQueue.PopFront(); + } + } + + // Job available? + if ( job.m_Function ) + { + // Execute job + (job.m_Function)( job.m_UserData ); + + // Immediately get another job. Ensure all jobs are consumed + // before shutting down. + continue; + } + + // Shutting down? + if ( m_WantToQuit.Load() ) + { + break; + } + + // Wait for work to be available + m_WakeSemaphore.Wait(); + } +} + +//------------------------------------------------------------------------------ +ThreadPool::ThreadPoolThread::ThreadPoolThread( uint32_t threadId, + ThreadPool & ownerPool ) + : m_ThreadId( threadId ) + , m_OwnerPool( ownerPool ) +{ + // Start thread + AStackString<> threadName; + threadName.Format( "ThreadPool_%02u", threadId ); + m_Thread.Start( ThreadFuncWrapper, threadName.Get(), this ); +} + +//------------------------------------------------------------------------------ +ThreadPool::ThreadPoolThread::~ThreadPoolThread() +{ + // Wait for thread to exit + m_Thread.Join(); +} + +//------------------------------------------------------------------------------ +void ThreadPool::ThreadPoolThread::ThreadFunc() +{ + PROFILE_FUNCTION; + + // First thread creates other threads + if ( ( m_ThreadId == 1 ) && ( m_OwnerPool.GetNumThreads() > 1 ) ) + { + PROFILE_SECTION("CreateAdditionalThreads"); + ThreadPoolThread * thread = this; + for ( uint32_t i = 2; i <= m_OwnerPool.GetNumThreads(); ++i ) + { + thread->m_Next = FNEW( ThreadPoolThread( i, m_OwnerPool ) ); + thread = thread->m_Next; + } + } + + // Process jobs + m_OwnerPool.ThreadFunc(); + + // First thread joins additional threads + if ( ( m_ThreadId == 1 ) && ( m_OwnerPool.GetNumThreads() > 1 ) ) + { + PROFILE_SECTION("DestroyAdditionalThreads"); + + // Join + ThreadPoolThread * thread = m_Next; + while ( thread ) + { + ThreadPoolThread * next = thread->m_Next; + FDELETE thread; + thread = next; + } + } +} + +//------------------------------------------------------------------------------ diff --git a/Code/Core/Process/ThreadPool.h b/Code/Core/Process/ThreadPool.h new file mode 100644 index 000000000..77989508a --- /dev/null +++ b/Code/Core/Process/ThreadPool.h @@ -0,0 +1,70 @@ +// ThreadPool +//------------------------------------------------------------------------------ +#pragma once + +// Includes +//------------------------------------------------------------------------------ +// Core +#include "Core/Containers/Array.h" +#include "Core/Containers/UniquePtr.h" +#include "Core/Process/Atomic.h" +#include "Core/Process/Mutex.h" +#include "Core/Process/Semaphore.h" +#include "Core/Process/Thread.h" + +// ThreadPool +//------------------------------------------------------------------------------ +class ThreadPool +{ +public: + explicit ThreadPool( uint32_t numThreads ); + ~ThreadPool(); + + // Enqueue jobs + using ThreadJobFunc = void(*)( void * param ); + void EnqueueJob( ThreadJobFunc func, void * userData = nullptr ); + + uint32_t GetNumThreads() const { return m_NumThreads; } + +protected: + void operator = (const ThreadPool & other) = delete; + + static uint32_t ThreadFuncWrapper( void * userData ); + void ThreadFunc(); + + // Job tracking structure + class ThreadJob + { + public: + ThreadJobFunc m_Function = nullptr; + void * m_UserData = nullptr; + }; + + // Thread structure + class ThreadPoolThread + { + public: + ThreadPoolThread( uint32_t threadId, ThreadPool & ownerPool ); + ~ThreadPoolThread(); + void ThreadFunc(); + + void operator = (const ThreadPoolThread & other) = delete; + protected: + uint32_t m_ThreadId = 0; + Thread m_Thread; + ThreadPoolThread * m_Next = nullptr; + ThreadPool & m_OwnerPool; + }; + + // Thread management + const uint32_t m_NumThreads = 0; + Atomic m_WantToQuit; + Semaphore m_WakeSemaphore; + ThreadPoolThread * m_FirstThread; + + // Job queue + Mutex m_JobQueueMutex; + Array< ThreadJob > m_JobQueue; +}; + +//------------------------------------------------------------------------------ diff --git a/Code/Core/Strings/AString.cpp b/Code/Core/Strings/AString.cpp index b2c11de05..5afb0bc99 100644 --- a/Code/Core/Strings/AString.cpp +++ b/Code/Core/Strings/AString.cpp @@ -709,7 +709,7 @@ void AString::TrimStart( char charToTrimFromStart ) uint32_t nbrCharsToRemoveFromStart = 0; const char * pos = m_Contents; const char * end = m_Contents + m_Length; - for ( ; pos < end && *pos == charToTrimFromStart; ++pos, ++nbrCharsToRemoveFromStart ) + for ( ; pos < end && *pos == charToTrimFromStart; ++pos, ++nbrCharsToRemoveFromStart ) { } @@ -723,7 +723,7 @@ void AString::TrimEnd( char charToTrimFromEnd ) uint32_t nbrCharsToRemoveFromEnd = 0; const char * pos = m_Contents + m_Length - 1; const char * end = m_Contents; - for ( ; pos >= end && *pos == charToTrimFromEnd; --pos, ++nbrCharsToRemoveFromEnd ) + for ( ; pos >= end && *pos == charToTrimFromEnd; --pos, ++nbrCharsToRemoveFromEnd ) { } diff --git a/Code/OSUI/OSWindow.cpp b/Code/OSUI/OSWindow.cpp index 677a742a3..69901c28d 100644 --- a/Code/OSUI/OSWindow.cpp +++ b/Code/OSUI/OSWindow.cpp @@ -283,7 +283,7 @@ void OSWindow::StartMessagePump() // Allow our message pump to do the work, whether it's WM_QUIT (0 bRet) or not (non-zero bRet) TranslateMessage( &msg ); - DispatchMessage( &msg ); + DispatchMessage( &msg ); } #elif defined( __OSX__ ) // This call blocks until messaged by StopMessagePump diff --git a/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff b/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff index b44292908..21c722148 100644 --- a/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff +++ b/Code/Tools/FBuild/BFFFuzzer/BFFFuzzer.bff @@ -53,6 +53,7 @@ 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' 'xxHash-Lib-$Platform$-$BuildConfigName$' + 'Zstd-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/bfffuzzer$ExeExtension$' .LinkerOptions + ' -pthread -ldl -lrt' diff --git a/Code/Tools/FBuild/Documentation/docs/changelog.html b/Code/Tools/FBuild/Documentation/docs/changelog.html index 2486d3c9f..531b4808f 100644 --- a/Code/Tools/FBuild/Documentation/docs/changelog.html +++ b/Code/Tools/FBuild/Documentation/docs/changelog.html @@ -26,6 +26,33 @@

Changelog

+
+ v1.11 (19th-Aug-2023) +
+
+ Notes +
    +
  • DB Version Changed : clean build will occur
    • +
+ Fixes +
    +
  • [Linux] Robustify file copy if sendfile isn't supported (Thanks to Nick Krecklow)
    • +
+ Improvements +
    +
  • Reduce cost of queueing jobs during graph traversal
    • +
  • Improve LightCache performance considerably for many CPU core systems
    • +
  • Directory listings with excluded folders optimized significantly
    • +
  • All directory listings optimized slightly
    • +
  • Database loading and various lookup performance improved slightly
    • +
  • Improve database load time slightly
    • +
  • Error #1100 emits location of previous declaration
    • +
  • -dbfile arg can explicitly set dependency db file location (Thanks to Troels Gram)
    • +
  • [Linux] Improve detection of max args length using _SC_ARG_MAX (Thanks to Domain)
    • +
  • VXCProject exposes AndroidApkLocation for Android Game Development Extension
    • +
+
+
v1.10 (28th-May-2023)
diff --git a/Code/Tools/FBuild/Documentation/docs/download.html b/Code/Tools/FBuild/Documentation/docs/download.html index ad96b027d..b871b67a9 100644 --- a/Code/Tools/FBuild/Documentation/docs/download.html +++ b/Code/Tools/FBuild/Documentation/docs/download.html @@ -39,22 +39,22 @@

Download

VersionWindowsOS XLinuxSourceMarkup
v1.10x64x64x64Zip | + GitHubNotePad++Visual StudioBash completion
v1.09 x64 x64
- - - - - + + + + +
VersionWindowsOS XLinuxSource
v1.10x64x64 & ARMx64Zip | - GitHub
v1.11x64x64 & ARMx64Zip | + GitHub
Syntax Highlighting @@ -87,6 +87,16 @@

Editor Integration

+ + + + + + + + + diff --git a/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html b/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html index b6cea32bd..ef830db49 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html @@ -135,6 +135,7 @@

VCXProject

.LinuxProjectType // (optional) Set the project type GUID for Linux .PackagePath, // (optional) Path to package to debug .AdditionalSymbolSearchPaths, // (optional) Additional symbol search paths for debugging +.AndroidApkLocation // (optional) Location of APK for Android Game Development Extension // Misc .PlatformToolset // (optional) Specify PlatformToolset. @@ -574,6 +575,9 @@

VCXProject

AdditionalSymbolSearchPaths - String - (Optional)

Path to additional symbols to be used when debugging.

+

AndroidApkLocation - String - (Optional)

+

Location of APK for Android Game Development Extension

+

PlatformToolset - String - (Optional)

Specify PlatformToolset.

diff --git a/Code/Tools/FBuild/Documentation/docs/home.html b/Code/Tools/FBuild/Documentation/docs/home.html index e766d5477..d55993e9d 100644 --- a/Code/Tools/FBuild/Documentation/docs/home.html +++ b/Code/Tools/FBuild/Documentation/docs/home.html @@ -38,14 +38,13 @@
- FASTBuild v1.10 released!  (28-May-2023) + FASTBuild v1.11 released!  (19-Aug-2023)
An updated version of FASTBuild can now be downloaded, featuring:
    -
  • Native support for Apple Silicon (ARM) Macs, significantly improving performance
    • -
  • Various performance enhancements
    • -
  • Some misc bug fixes and reliability improvements
    • +
  • Several general performance enhancements
    • +
  • Various small bug fixes and reliability improvements
  • Details...
diff --git a/Code/Tools/FBuild/Documentation/docs/options.html b/Code/Tools/FBuild/Documentation/docs/options.html index 3440cda31..332c06eae 100644 --- a/Code/Tools/FBuild/Documentation/docs/options.html +++ b/Code/Tools/FBuild/Documentation/docs/options.html @@ -64,13 +64,17 @@

FBuild.exe

- + + + + + @@ -331,20 +335,26 @@

FBuild.exe Detailed

Instead of building specified targets generate a JSON compilation database for them. Resulting compilation database will include entries for all source files from ObjectList and Library nodes that are dependencies of the specified targets.

-
-config [path]
+
-config <path>

Explicitly specify the config file to use. By default, FASTBuild looks for "fbuild.bff" in the current directory. This options allows a file to be explicitly specified instead.

-
--continueafterdbmove
+
-continueafterdbmove

Allow build to continue after a DB move.

FASTBuild's database is tied to the directory in which it was created and cannot be moved. If a move is detected, an error will be emitted. -continueafterdbmove allows the build to continue after this error has been emitted, ignoring and replacing the DB file.

-
-debug
+
-dbfile <path>
+
+

Explicitly specify the dependency database file to use. By default, FASTBuild will load and save its dependency database in the same directory as the config file +(with a ".platform.fdb" suffix). This option allows the file to be explicitly specified instead.

+
+ +
-debug

[Windows Only] Display a message box on startup to allow a debugger to be attached. Can be useful if triaging problems with FASTBuild that can't be reproduced in the debugger.

diff --git a/Code/Tools/FBuild/FBuild/FBuild.bff b/Code/Tools/FBuild/FBuild/FBuild.bff index f3c260a6c..79f93147b 100644 --- a/Code/Tools/FBuild/FBuild/FBuild.bff +++ b/Code/Tools/FBuild/FBuild/FBuild.bff @@ -50,6 +50,7 @@ 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' 'xxHash-Lib-$Platform$-$BuildConfigName$' + 'Zstd-Lib-$Platform$-$BuildConfigName$' } #if __LINUX__ .LinkerOutput = '$OutputBase$/$ProjectPath$/fbuild$ExeExtension$' // NOTE: lower case @@ -106,7 +107,7 @@ } } #endif - + // Aliases //-------------------------------------------------------------------------- CreateCommonAliases( .ProjectName ) diff --git a/Code/Tools/FBuild/FBuild/Main.cpp b/Code/Tools/FBuild/FBuild/Main.cpp index 9f2cb9f22..356c1d933 100644 --- a/Code/Tools/FBuild/FBuild/Main.cpp +++ b/Code/Tools/FBuild/FBuild/Main.cpp @@ -256,7 +256,7 @@ int Main( int argc, char * argv[] ) if ( problemSavingBuildProfileJSON ) { return FBUILD_FAILED_TO_WRITE_PROFILE_JSON; - } + } return ( result == true ) ? FBUILD_OK : FBUILD_BUILD_FAILED; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFBooleanExpParser.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFBooleanExpParser.cpp index a924bcf74..9a3fb9ec8 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFBooleanExpParser.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFBooleanExpParser.cpp @@ -770,7 +770,7 @@ static bool ParseBinaryBooleanExp( const Function * function, bool lhs, BFFToken // ParseIntComparisonExp // Forms: -// bool-exp = +// bool-exp = // | number-exp ==/!=/>/>=/GetValueString().EndsWith( *pos ) ); - ASSERT( iter->GetValueString().GetLength() >= 4 ); // at least one char inside: ."x" + ASSERT( iter->GetValueString().GetLength() >= 4 ); // at least one char inside: ."x" // unescape and subsitute embedded variables AStackString<> value; @@ -222,7 +222,7 @@ bool BFFParser::Parse( BFFTokenRange & iter ) } // sanity check it is a sensible length - if ( name.GetLength() + 1 > MAX_VARIABLE_NAME_LENGTH ) // +1 for '.' will be added + if ( name.GetLength() + 1 > MAX_VARIABLE_NAME_LENGTH ) // +1 for '.' will be added { Error::Error_1014_VariableNameIsTooLong( iter, (uint32_t)value.GetLength(), (uint32_t)MAX_VARIABLE_NAME_LENGTH ); return false; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp index 539eaf478..0ce6eeadb 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.cpp @@ -132,7 +132,7 @@ void BFFStackFrame::DisconnectStackChain() // SetVarInt //------------------------------------------------------------------------------ -/*static*/ void BFFStackFrame::SetVarInt( const AString & name, +/*static*/ void BFFStackFrame::SetVarInt( const AString & name, const BFFToken & token, int value, BFFStackFrame * frame ) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h index 73518f98d..8dcf9d15e 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFStackFrame.h @@ -95,7 +95,7 @@ class BFFStackFrame const AString & GetLastVariableSeen() const { return m_LastVariableSeen; } BFFStackFrame * GetLastVariableSeenFrame() const { return m_LastVariableSeenFrame; } void SetLastVariableSeen( const AString & varName, BFFStackFrame * frame ) - { + { m_LastVariableSeen = varName; m_LastVariableSeenFrame = frame; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp index c0951e8ee..3aac50c63 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp @@ -210,6 +210,7 @@ Function::~Function() = default; { return false; // substitution will have emitted an error } + m_AliasForFunctionSourceToken = headerArgsIter; ++headerArgsIter; } else if ( headerArgsIter->IsVariable() ) @@ -247,6 +248,7 @@ Function::~Function() = default; // Store alias name for use in Commit m_AliasForFunction = varSrc->GetString(); + m_AliasForFunctionSourceToken = headerArgsIter; ++headerArgsIter; } @@ -315,29 +317,32 @@ Function::~Function() = default; // - For nodes that specify a name as a property (usually the output of the node) // use that as the name (i.e. the filename is the name) // - Otherwise, what would normally be the alias - AStackString<> nameFromMetaData; + AString nameFromMetaData; if ( GetNameForNode( nodeGraph, funcStartIter, node->GetReflectionInfoV(), nameFromMetaData ) == false ) { FDELETE node; return false; // GetNameForNode will have emitted an error } const bool aliasUsedForName = nameFromMetaData.IsEmpty(); - const AString & name = ( aliasUsedForName ) ? m_AliasForFunction : nameFromMetaData; + AString & name = ( aliasUsedForName ) ? m_AliasForFunction : nameFromMetaData; ASSERT( name.IsEmpty() == false ); + const BFFToken * nameSourceToken = aliasUsedForName ? m_AliasForFunctionSourceToken + : funcStartIter; // TODO:C This could probably be improved // Check name isn't already used - if ( nodeGraph.FindNode( name ) ) + if ( const Node * existingNode = nodeGraph.FindNode( name ) ) { - Error::Error_1100_AlreadyDefined( funcStartIter, this, name ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( nameSourceToken, this, name, existingToken ); FDELETE node; return false; } // Set Name - node->SetName( name ); + node->SetName( Move( name ) ); // Register with NodeGraph - nodeGraph.RegisterNode( node ); + nodeGraph.RegisterNode( node, nameSourceToken ); // Set properties if ( !PopulateProperties( nodeGraph, funcStartIter, node ) ) @@ -358,7 +363,7 @@ Function::~Function() = default; } // handle alias creation - return ProcessAlias( nodeGraph, funcStartIter, node ); + return ProcessAlias( nodeGraph, node ); } // GetString @@ -562,8 +567,8 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, Node * node = nodeGraph.FindNode( name ); if ( node == nullptr ) { - node = nodeGraph.CreateDirectoryListNode( name ); - DirectoryListNode * dln = node->CastTo< DirectoryListNode >(); + DirectoryListNode * dln = nodeGraph.CreateNode( name, iter ); + node = dln; dln->m_Path = path; if ( patterns ) { @@ -645,7 +650,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, } // Implicitly create the new node - compilerNode = nodeGraph.CreateCompilerNode( nodeName ); + compilerNode = nodeGraph.CreateNode( nodeName, iter ); VERIFY( compilerNode->GetReflectionInfoV()->SetProperty( compilerNode, "Executable", compiler ) ); VERIFY( compilerNode->GetReflectionInfoV()->SetProperty( compilerNode, "AllowDistribution", false ) ); const char * lastSlash = compiler.FindLast( NATIVE_SLASH ); @@ -676,7 +681,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, Node * node = nodeGraph.FindNode( file ); if ( node == nullptr ) { - node = nodeGraph.CreateFileNode( file ); + node = nodeGraph.CreateNode( file, iter ); } else if ( node->IsAFile() == false ) { @@ -786,7 +791,7 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, if ( n == nullptr ) { // not found - create a new file node - n = nodeGraph.CreateFileNode( nodeName ); + n = nodeGraph.CreateNode( nodeName, iter ); nodes.Add( n ); return true; } @@ -905,16 +910,16 @@ bool Function::GetStrings( const BFFToken * iter, Array< AString > & strings, co // ProcessAlias //------------------------------------------------------------------------------ -bool Function::ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Node * nodeToAlias ) const +bool Function::ProcessAlias( NodeGraph & nodeGraph, Node * nodeToAlias ) const { Dependencies nodesToAlias( 1 ); nodesToAlias.Add( nodeToAlias ); - return ProcessAlias( nodeGraph, iter, nodesToAlias ); + return ProcessAlias( nodeGraph, nodesToAlias ); } // ProcessAlias //------------------------------------------------------------------------------ -bool Function::ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Dependencies & nodesToAlias ) const +bool Function::ProcessAlias( NodeGraph & nodeGraph, Dependencies & nodesToAlias ) const { if ( m_AliasForFunction.IsEmpty() ) { @@ -922,18 +927,20 @@ bool Function::ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Depen } // check for duplicates - if ( nodeGraph.FindNode( m_AliasForFunction ) ) + if ( const Node * existingNode = nodeGraph.FindNode( m_AliasForFunction ) ) { - Error::Error_1100_AlreadyDefined( iter, this, m_AliasForFunction ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( m_AliasForFunctionSourceToken, this, m_AliasForFunction, existingToken ); return false; } // create an alias against the node - AliasNode * an = nodeGraph.CreateAliasNode( m_AliasForFunction ); + AliasNode * an = nodeGraph.CreateNode( m_AliasForFunction, m_AliasForFunctionSourceToken ); an->m_StaticDependencies = nodesToAlias; // TODO: make this use m_Targets & Initialize() - // clear the string so it can't be used again + // Clear the alias info so it can't be used again m_AliasForFunction.Clear(); + m_AliasForFunctionSourceToken = nullptr; return true; } @@ -982,9 +989,10 @@ bool Function::GetNameForNode( NodeGraph & nodeGraph, const BFFToken * iter, con } // Check that name isn't already used - if ( nodeGraph.FindNode( strings[0] ) ) + if ( const Node * existingNode = nodeGraph.FindNode( strings[0] ) ) { - Error::Error_1100_AlreadyDefined( iter, this, strings[0] ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( iter, this, strings[0], existingToken ); return false; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.h b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.h index 1244ceb76..24464e7b6 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.h @@ -126,6 +126,7 @@ class Function // for functions that support a simple alias parameter, the base class can // parse it out mutable AString m_AliasForFunction; + mutable const BFFToken * m_AliasForFunctionSourceToken = nullptr; // Helpers to get nodes bool GetNodeList( NodeGraph & nodeGraph, @@ -145,8 +146,8 @@ class Function bool GetStrings( const BFFToken * iter, Array< AString > & strings, const char * name, bool required = false ) const; // helper function to make alias for target - bool ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Node * nodeToAlias ) const; - bool ProcessAlias( NodeGraph & nodeGraph, const BFFToken * iter, Dependencies & nodesToAlias ) const; + bool ProcessAlias( NodeGraph & nodeGraph, Node * nodeToAlias ) const; + bool ProcessAlias( NodeGraph & nodeGraph, Dependencies & nodesToAlias ) const; // Reflection based property population bool GetNameForNode( NodeGraph & nodeGraph, const BFFToken *iter, const ReflectionInfo * ri, AString & name ) const; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp index 4a62b419e..0cc86e29c 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp @@ -104,7 +104,7 @@ FunctionCopy::FunctionCopy() else { // source file not defined by use - assume an external file - srcNodes.Append( nodeGraph.CreateFileNode( *it ) ); + srcNodes.Append( nodeGraph.CreateNode( *it, funcStartIter ) ); } } } @@ -148,14 +148,15 @@ FunctionCopy::FunctionCopy() } // check node doesn't already exist - if ( nodeGraph.FindNode( dst ) ) + if ( const Node * existingNode = nodeGraph.FindNode( dst ) ) { - Error::Error_1100_AlreadyDefined( funcStartIter, this, dst ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( funcStartIter, this, dst, existingToken ); return false; } // create our node - CopyFileNode * copyFileNode = nodeGraph.CreateCopyFileNode( dst ); + CopyFileNode * copyFileNode = nodeGraph.CreateNode( dst, funcStartIter ); copyFileNode->m_Source = srcNode->GetName(); copyFileNode->m_PreBuildDependencyNames = preBuildDependencyNames; if ( !copyFileNode->Initialize( nodeGraph, funcStartIter, this ) ) @@ -167,7 +168,7 @@ FunctionCopy::FunctionCopy() } // handle alias creation - return ProcessAlias( nodeGraph, funcStartIter, copyNodes ); + return ProcessAlias( nodeGraph, copyNodes ); } // GetSourceNodes diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionIf.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionIf.cpp index 53c5df40b..f89555b84 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionIf.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionIf.cpp @@ -42,12 +42,12 @@ FunctionIf::FunctionIf() // Iterate the args const BFFTokenRange header( headerRange ); - bool headerResult; + bool headerResult; if ( !BFFBooleanExpParser::Parse( this, header, headerResult ) ) { return false; } - + if ( !headerResult ) { return true; diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.cpp index fd9f9f312..25e9a1b4f 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.cpp @@ -229,8 +229,8 @@ bool FunctionObjectList::CheckMSVCPCHFlags_Use( const BFFToken * iter, // GetExtraOutputPaths //------------------------------------------------------------------------------ -void FunctionObjectList::GetExtraOutputPaths( const AString & args, - AString & outPDBPath, +void FunctionObjectList::GetExtraOutputPaths( const AString & args, + AString & outPDBPath, AString & outASMPath, AString & outSourceDependenciesPath ) { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.h b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.h index fc6343595..de34c9048 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.h +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionObjectList.h @@ -40,7 +40,7 @@ class FunctionObjectList : public Function friend class TestObjectList; static void GetExtraOutputPaths( const AString & args, - AString & outPDBPath, + AString & outPDBPath, AString & outASMPath, AString & outSourceDependenciesPath ); static void GetExtraOutputPath( const AString * it, const AString * end, const char * option, AString & path ); diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp index 7a4aacfc6..076e81a19 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionRemoveDir.cpp @@ -36,13 +36,14 @@ FunctionRemoveDir::FunctionRemoveDir() //------------------------------------------------------------------------------ /*virtual*/ bool FunctionRemoveDir::Commit( NodeGraph & nodeGraph, const BFFToken * funcStartIter ) const { - if ( nodeGraph.FindNode( m_AliasForFunction ) ) + if ( const Node * existingNode = nodeGraph.FindNode( m_AliasForFunction ) ) { - Error::Error_1100_AlreadyDefined( funcStartIter, this, m_AliasForFunction ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( funcStartIter, this, m_AliasForFunction, existingToken ); return false; } - RemoveDirNode * removeDirNode = nodeGraph.CreateRemoveDirNode( m_AliasForFunction ); + Node * removeDirNode = nodeGraph.CreateNode( m_AliasForFunction, funcStartIter ); if ( !PopulateProperties( nodeGraph, funcStartIter, removeDirNode ) ) { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp index f8ba7309e..b9309475b 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionSettings.cpp @@ -28,13 +28,14 @@ FunctionSettings::FunctionSettings() /*virtual*/ bool FunctionSettings::Commit( NodeGraph & nodeGraph, const BFFToken * funcStartIter ) const { AStackString<> name( "$$Settings$$" ); - if ( nodeGraph.FindNode( name ) ) + if ( const Node * existingNode = nodeGraph.FindNode( name ) ) { - Error::Error_1100_AlreadyDefined( funcStartIter, this, name ); + const BFFToken * existingToken = nodeGraph.FindNodeSourceToken( existingNode ); + Error::Error_1100_AlreadyDefined( funcStartIter, this, name, existingToken ); return false; } - SettingsNode * settingsNode = nodeGraph.CreateSettingsNode( name ); + SettingsNode * settingsNode = nodeGraph.CreateNode( name, funcStartIter ); if ( !PopulateProperties( nodeGraph, funcStartIter, settingsNode ) ) { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp index 32ed62412..1010176f7 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Tokenizer/BFFTokenizer.cpp @@ -694,16 +694,16 @@ bool BFFTokenizer::HandleDirective_If( const BFFFile & file, const char * & pos, // Store to history operatorHistory[ numOperators++ ] = r; - + // Check for excessive complexity if ( numOperators == BFFParser::MAX_OPERATOR_HISTORY ) { Error::Error_1047_IfExpressionTooComplex( argsIter.GetCurrent() ); return false; - } + } } } - + // Apply any && operators. Any valid expression isn't going to end with an operator, so we don't check that. for ( uint32_t i = 0; i < ( numOperators - 1 ); i++ ) { diff --git a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp index 1c379ccfa..b4b4585ac 100644 --- a/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp +++ b/Code/Tools/FBuild/FBuildCore/Cache/LightCache.cpp @@ -41,8 +41,8 @@ class IncludedFile class Include { public: - Include( const AString & include, IncludeType type ) - : m_Include( include ) + Include( AString && include, IncludeType type ) + : m_Include( Move( include ) ) , m_Type( type ) {} @@ -97,8 +97,7 @@ class IncludedFileHashSet return nullptr; } - // If two threads find the same include simultaneously, we delete the new - // one and return the old one. + // Insert a new item. Item must not already exist. const IncludedFile * Insert( IncludedFile * item ) { if ( ( m_Buckets.GetSize() / 2 ) <= m_Elts ) @@ -111,13 +110,7 @@ class IncludedFileHashSet IncludedFile ** location = InternalFind( item->m_FileName, item->m_FileNameHash ); ASSERT( location != nullptr ); - if ( *location != nullptr ) - { - // A race between multiple threads got us a duplicate item. - // delete the new one - FDELETE item; - return *location; - } + ASSERT( *location == nullptr ); // Item must not exist already ++m_Elts; *location = item; @@ -227,7 +220,8 @@ IncludedFile::~IncludedFile() // IncludedFileBucket //------------------------------------------------------------------------------ -class IncludedFileBucket +PRAGMA_DISABLE_PUSH_MSVC( 4324 ) // structure was padded due to alignment specifier +class alignas(64) IncludedFileBucket // Align to cache line boundary { public: IncludedFileBucket() = default; @@ -241,9 +235,9 @@ class IncludedFileBucket Mutex m_Mutex; IncludedFileHashSet m_HashSet; }; -// using a power of two number of buckets. 64 top level buckets should be a -// reasonable tradeoff between size and contention -#define LIGHTCACHE_NUM_BUCKET_BITS 6 +PRAGMA_DISABLE_POP_MSVC // 4324 +// Power of two number of buckets +#define LIGHTCACHE_NUM_BUCKET_BITS 9 // 512 buckets #define LIGHTCACHE_NUM_BUCKETS ( 1ULL << LIGHTCACHE_NUM_BUCKET_BITS ) #define LIGHTCACHE_BUCKET_MASK_BASE ( LIGHTCACHE_NUM_BUCKETS - 1ULL ) // use upper bits for bucket selection, as lower bits get used in the hash set @@ -446,7 +440,7 @@ bool LightCache::ParseDirective_Include( IncludedFile & file, const char * & pos SkipWhitespace( pos ); // Get include string - AStackString<> include; + AString include; if ( ( *pos == '"' ) || ( *pos == '<' ) ) { // Looks like a normal include @@ -458,12 +452,12 @@ bool LightCache::ParseDirective_Include( IncludedFile & file, const char * & pos return false; } - file.m_Includes.EmplaceBack( include, includeType ); + file.m_Includes.EmplaceBack( Move( include ), includeType ); return true; } // Not a normal include - perhaps this is a macro? - AStackString<> macroName; + AString macroName; if ( ParseMacroName( pos, macroName ) == false ) { // We saw an unexpected sequence after the #include @@ -472,7 +466,7 @@ bool LightCache::ParseDirective_Include( IncludedFile & file, const char * & pos } // Store the macro include which will be resolved later - file.m_Includes.EmplaceBack( macroName, IncludeType::MACRO ); + file.m_Includes.EmplaceBack( Move( macroName ), IncludeType::MACRO ); return true; } @@ -600,7 +594,7 @@ bool LightCache::ParseMacroName( const char * & pos, AString & outMacroName ) c = *pos; if ( ( ( c >= 'a' ) && ( c <= 'z' ) ) || ( ( c >= 'A' ) && ( c <= 'Z' ) ) || - ( ( c >= '0' ) && ( c <= '9' ) ) || + ( ( c >= '0' ) && ( c <= '9' ) ) || ( c == '_' ) ) { ++pos; @@ -861,21 +855,34 @@ const IncludedFile * LightCache::FileExists( const AString & fileName ) const uint64_t fileNameHash = xxHash3::Calc64( fileName ); const uint64_t bucketIndex = LIGHTCACHE_HASH_TO_BUCKET( fileNameHash ); IncludedFileBucket & bucket = g_AllIncludedFiles[ bucketIndex ]; + + // Lock the bucket while we deal with this file. + // + // This prevents two threads processing the same file, which for most + // codebases would otherwise happen a lot (code tends to have a large set + // of shared headers). + // + // Collisions in the bucketIndex mean some unrelated files will block + // each other, but not redundantly processing the same file results + // in a significant speedup which more than offsets the blocking caused + // by collisions. LIGHTCACHE_NUM_BUCKET_BITS can be increased to further + // reduce collisions at the cost of memory. + // + MutexHolder mh( bucket.m_Mutex ); + // Retrieve from shared cache { - MutexHolder mh( bucket.m_Mutex ); const IncludedFile * location = bucket.m_HashSet.Find( fileName, fileNameHash ); if ( location ) { m_IncludeDefines.Append( location->m_IncludeDefines ); - + return location; // File previously handled so we can re-use the result } } // A newly seen file IncludedFile * newFile = FNEW( IncludedFile() ); - const IncludedFile * retval = nullptr; newFile->m_FileNameHash = fileNameHash; newFile->m_FileName = fileName; newFile->m_Exists = false; @@ -885,23 +892,16 @@ const IncludedFile * LightCache::FileExists( const AString & fileName ) FileStream f; if ( f.Open( fileName.Get() ) == false ) { - { - // Store to shared cache - MutexHolder mh( bucket.m_Mutex ); - retval = bucket.m_HashSet.Insert( newFile ); - } - return retval; + // Store to shared cache + return bucket.m_HashSet.Insert( newFile ); } // File exists - parse it newFile->m_Exists = true; Parse( newFile, f ); - { - // Store to shared cache - MutexHolder mh( bucket.m_Mutex ); - retval = bucket.m_HashSet.Insert( newFile ); - } + // Store to shared cache + const IncludedFile * retval = bucket.m_HashSet.Insert( newFile ); m_IncludeDefines.Append( retval->m_IncludeDefines ); diff --git a/Code/Tools/FBuild/FBuildCore/Error.cpp b/Code/Tools/FBuild/FBuildCore/Error.cpp index 0352daa15..4b2dc15c1 100644 --- a/Code/Tools/FBuild/FBuildCore/Error.cpp +++ b/Code/Tools/FBuild/FBuildCore/Error.cpp @@ -477,10 +477,15 @@ //------------------------------------------------------------------------------ /*static*/ void Error::Error_1100_AlreadyDefined( const BFFToken * iter, const Function * function, - const AString & name ) + const AString & name, + const BFFToken * previousDeclarationToken ) { FormatError( iter, 1100u, function, "Target '%s' already defined.", name.Get() ); + if ( previousDeclarationToken ) + { + FormatError( previousDeclarationToken, 1100u, nullptr, "Previously declared here:" ); + } } // Error_1101_MissingProperty @@ -765,7 +770,7 @@ "1234567890" "~`!@#$%^&*()-_+=" "{[]}|\\" - ";:\"'" + ";:\"'" "<,.>/?"; bool printable = false; const char c = *iter->GetSourcePos(); diff --git a/Code/Tools/FBuild/FBuildCore/Error.h b/Code/Tools/FBuild/FBuildCore/Error.h index 2556aa0c1..fa38ddf2a 100644 --- a/Code/Tools/FBuild/FBuildCore/Error.h +++ b/Code/Tools/FBuild/FBuildCore/Error.h @@ -138,7 +138,8 @@ class Error //------------------------------------------------------------------------------ static void Error_1100_AlreadyDefined( const BFFToken * iter, const Function * function, - const AString & name ); + const AString & name, + const BFFToken * previousDeclarationToken = nullptr ); static void Error_1101_MissingProperty( const BFFToken * iter, const Function * function, const AString & name ); diff --git a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriverBase.h b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriverBase.h index 9c776efbf..fda1c8549 100644 --- a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriverBase.h +++ b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriverBase.h @@ -1,4 +1,4 @@ -// CompilerDriverBase.h +// CompilerDriverBase.h //------------------------------------------------------------------------------ #pragma once diff --git a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.cpp b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.cpp index dcd56cb33..4b83f639b 100644 --- a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.cpp +++ b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.cpp @@ -143,13 +143,13 @@ CompilerDriver_CL::~CompilerDriver_CL() = default; { ++index; // Skip next arg which specifies the mode for '/sourceDependencies' return true; - } + } // Remove "/sourceDependencies" if ( IsStartOfCompilerArg_MSVC( token, "sourceDependencies" ) ) { return true; } - + return false; } diff --git a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.h b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.h index 66d0517ef..55a12af63 100644 --- a/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.h +++ b/Code/Tools/FBuild/FBuildCore/ExeDrivers/Compiler/CompilerDriver_CL.h @@ -1,4 +1,4 @@ -// CompilerDriver_CL.h +// CompilerDriver_CL.h //------------------------------------------------------------------------------ #pragma once diff --git a/Code/Tools/FBuild/FBuildCore/FBuild.cpp b/Code/Tools/FBuild/FBuildCore/FBuild.cpp index fa3b41aac..2412e9c17 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuild.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuild.cpp @@ -34,6 +34,7 @@ #include "Core/Mem/SmallBlockAllocator.h" #include "Core/Process/Atomic.h" #include "Core/Process/SystemMutex.h" +#include "Core/Process/ThreadPool.h" #include "Core/Profile/Profile.h" #include "Core/Strings/AStackString.h" #include "Core/Tracing/Tracing.h" @@ -78,6 +79,12 @@ FBuild::FBuild( const FBuildOptions & options ) // store all user provided options m_Options = options; + // Create ThreadPool + if ( m_Options.m_NumWorkerThreads > 0 ) + { + m_ThreadPool = FNEW( ThreadPool( m_Options.m_NumWorkerThreads ) ); + } + // track the old working dir to restore if modified (mainly for unit tests) VERIFY( FileIO::GetCurrentDir( m_OldWorkingDir ) ); @@ -129,6 +136,8 @@ FBuild::~FBuild() { FDELETE( &BuildProfiler::Get() ); } + + FDELETE m_ThreadPool; } // Initialize @@ -154,26 +163,30 @@ bool FBuild::Initialize( const char * nodeGraphDBFile ) } else { - m_DependencyGraphFile = bffFile; - if ( m_DependencyGraphFile.EndsWithI( ".bff" ) ) + if ( m_Options.m_DBFile.IsEmpty() ) + { + m_DependencyGraphFile = bffFile; + if ( m_DependencyGraphFile.EndsWithI( ".bff" ) ) + { + m_DependencyGraphFile.SetLength( m_DependencyGraphFile.GetLength() - 4 ); + } + #if defined( __WINDOWS__ ) + m_DependencyGraphFile += ".windows.fdb"; + #elif defined( __OSX__ ) + m_DependencyGraphFile += ".osx.fdb"; + #elif defined( __LINUX__ ) + m_DependencyGraphFile += ".linux.fdb"; + #endif + } + else { - m_DependencyGraphFile.SetLength( m_DependencyGraphFile.GetLength() - 4 ); + // DB filename explicitly set on command line + m_DependencyGraphFile = m_Options.m_DBFile; } - #if defined( __WINDOWS__ ) - m_DependencyGraphFile += ".windows.fdb"; - #elif defined( __OSX__ ) - m_DependencyGraphFile += ".osx.fdb"; - #elif defined( __LINUX__ ) - m_DependencyGraphFile += ".linux.fdb"; - #endif } - SmallBlockAllocator::SetSingleThreadedMode( true ); - m_DependencyGraph = NodeGraph::Initialize( bffFile, m_DependencyGraphFile.Get(), m_Options.m_ForceDBMigration_Debug ); - SmallBlockAllocator::SetSingleThreadedMode( false ); - if ( m_DependencyGraph == nullptr ) { return false; @@ -369,7 +382,7 @@ void FBuild::SaveDependencyGraph( MemoryStream & stream, const char* nodeGraphDB AtomicStoreRelaxed( &s_AbortBuild, false ); // allow multiple runs in same process // create worker threads - m_JobQueue = FNEW( JobQueue( m_Options.m_NumWorkerThreads ) ); + m_JobQueue = FNEW( JobQueue( m_Options.m_NumWorkerThreads, m_ThreadPool ) ); // create the connection management system if needed // (must be after JobQueue is created) @@ -796,7 +809,7 @@ bool FBuild::GenerateDotGraph( const Array< AString > & targets, const bool full OUTPUT( "Saving DOT graph file to '%s'\n", dotFileName ); // Generate - AString buffer( 10 * 1024 * 1024 ); + AString buffer( 10 * 1024 * 1024 ); m_DependencyGraph->SerializeToDotFormat( deps, fullGraph, buffer ); // Write to disk diff --git a/Code/Tools/FBuild/FBuildCore/FBuild.h b/Code/Tools/FBuild/FBuildCore/FBuild.h index d7028434f..2898fb52d 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuild.h +++ b/Code/Tools/FBuild/FBuildCore/FBuild.h @@ -27,6 +27,7 @@ class MemoryStream; class JobQueue; class Node; class NodeGraph; +class ThreadPool; // FBuild //------------------------------------------------------------------------------ @@ -122,6 +123,7 @@ class FBuild : public Singleton< FBuild > static volatile bool s_AbortBuild; // -fastcancel - TODO:C merge with StopBuild NodeGraph * m_DependencyGraph; + ThreadPool * m_ThreadPool = nullptr; JobQueue * m_JobQueue; mutable Mutex m_ClientLifetimeMutex; Client * m_Client; // manage connections to worker servers diff --git a/Code/Tools/FBuild/FBuildCore/FBuildCore.bff b/Code/Tools/FBuild/FBuildCore/FBuildCore.bff index 54232787c..a5d945afd 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildCore.bff +++ b/Code/Tools/FBuild/FBuildCore/FBuildCore.bff @@ -31,6 +31,7 @@ // Extra Compiler Options .CompilerOptions + .LZ4IncludePaths + + .ZstdIncludePaths .CompilerOptions + ' "-I../External/VSProjTypeExtractor"' // Output diff --git a/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp b/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp index 437e5a816..ece79c584 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuildOptions.cpp @@ -167,7 +167,7 @@ FBuildOptions::OptionsResult FBuildOptions::ProcessCommandLine( int argc, char * } m_CacheCompressionLevel = static_cast< int16_t >( cacheCompressionLevel ); i++; // skip extra arg we've consumed - + // add to args we might pass to subprocess m_Args += ' '; m_Args += argv[ sizeIndex ]; @@ -222,6 +222,25 @@ FBuildOptions::OptionsResult FBuildOptions::ProcessCommandLine( int argc, char * m_Args += '"'; continue; } + else if ( thisArg == "-dbfile" ) + { + const int32_t pathIndex = ( i + 1 ); + if ( pathIndex >= argc ) + { + OUTPUT( "FBuild: Error: Missing for '-dbfile' argument\n" ); + OUTPUT( "Try \"%s -help\"\n", programName.Get() ); + return OPTIONS_ERROR; + } + m_DBFile = argv[ pathIndex ]; + i++; // skip extra arg we've consumed + + // add to args we might pass to subprocess + m_Args += ' '; + m_Args += '"'; // surround db file with quotes to avoid problems with spaces in the path + m_Args += m_DBFile; + m_Args += '"'; + continue; + } #if defined( __WINDOWS__ ) else if ( thisArg == "-debug" ) { @@ -356,7 +375,7 @@ FBuildOptions::OptionsResult FBuildOptions::ProcessCommandLine( int argc, char * thisArg.Tokenize( reportTokens, '=' ); // if there is something after the '=' sign, then we take whatever comes after as the report type - if ( reportTokens.GetSize() > 1 ) + if ( reportTokens.GetSize() > 1 ) { m_ReportType = reportTokens[ 1 ]; m_ReportType.ToLower(); @@ -624,6 +643,7 @@ void FBuildOptions::DisplayHelp( const AString & programName ) const " -config Explicitly specify the config file to use.\n" " -continueafterdbmove\n" " Allow builds after a DB move.\n" + " -dbfile Explicitly specify the dependency database file to use.\n" " -debug (Windows) Break at startup, to attach debugger.\n" " -dist Allow distributed compilation.\n" " -distverbose Print detailed info for distributed compilation.\n" diff --git a/Code/Tools/FBuild/FBuildCore/FBuildOptions.h b/Code/Tools/FBuild/FBuildCore/FBuildOptions.h index a37086833..1a107fb37 100755 --- a/Code/Tools/FBuild/FBuildCore/FBuildOptions.h +++ b/Code/Tools/FBuild/FBuildCore/FBuildOptions.h @@ -99,6 +99,7 @@ class FBuildOptions bool m_FixupErrorPaths = false; bool m_ForceDBMigration_Debug = false; // Force migration even if bff has not changed (for tests) bool m_ContinueAfterDBMove = false; + AString m_DBFile; uint32_t m_NumWorkerThreads = 0; // True default detected in constructor AString m_ConfigFile; diff --git a/Code/Tools/FBuild/FBuildCore/FBuildVersion.h b/Code/Tools/FBuild/FBuildCore/FBuildVersion.h index d227ef6a4..3c4c21386 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuildVersion.h +++ b/Code/Tools/FBuild/FBuildCore/FBuildVersion.h @@ -4,8 +4,8 @@ // Defines //------------------------------------------------------------------------------ -#define FBUILD_VERSION_STRING "v1.10" -#define FBUILD_VERSION (uint32_t)110 +#define FBUILD_VERSION_STRING "v1.11" +#define FBUILD_VERSION (uint32_t)111 #if defined( __WINDOWS__ ) #define FBUILD_VERSION_PLATFORM "Windows" #elif defined( __APPLE__ ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/AliasNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/AliasNode.cpp index f53b2a483..baa3d4477 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/AliasNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/AliasNode.cpp @@ -20,8 +20,9 @@ REFLECT_END( AliasNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ AliasNode::AliasNode() - : Node( AString::GetEmpty(), Node::ALIAS_NODE, Node::FLAG_ALWAYS_BUILD ) + : Node( Node::ALIAS_NODE ) { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 1; // almost no work is done for this node } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp index d98ef4bd1..9e15eb50c 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp @@ -42,7 +42,7 @@ REFLECT_END( CSNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ CSNode::CSNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_CompilerInputPathRecurse( true ) , m_NumCompilerInputFiles( 0 ) , m_NumCompilerReferences( 0 ) @@ -127,7 +127,7 @@ CSNode::~CSNode() = default; // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool CSNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool /*forceClean*/ ) +/*virtual*/ bool CSNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { // clear dynamic deps from previous passes m_DynamicDependencies.Clear(); @@ -151,7 +151,7 @@ CSNode::~CSNode() = default; Node * sn = nodeGraph.FindNode( file.m_Name ); if ( sn == nullptr ) { - sn = nodeGraph.CreateFileNode( file.m_Name ); + sn = nodeGraph.CreateNode( file.m_Name ); } else if ( sn->IsAFile() == false ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.h b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.h index 5af00a494..b043e3ad2 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.h @@ -25,7 +25,7 @@ class CSNode : public FileNode static inline Node::Type GetTypeS() { return Node::CS_NODE; } private: - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; CompilerNode * GetCompiler() const; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp index d552e1e85..b97dbbe4c 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CompilerNode.cpp @@ -43,7 +43,7 @@ REFLECT_END( CompilerNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ CompilerNode::CompilerNode() - : Node( AString::GetEmpty(), Node::COMPILER_NODE, Node::FLAG_NONE ) + : Node( Node::COMPILER_NODE ) , m_AllowDistribution( true ) , m_AllowResponseFile( false ) , m_ForceResponseFile( false ) @@ -308,7 +308,7 @@ bool CompilerNode::InitializeCompilerFamily( const BFFToken * iter, const Functi { m_CompilerFamilyEnum = CLANG_CL; return true; - } + } if ( m_CompilerFamilyString.EqualsI( "snc" ) ) { m_CompilerFamilyEnum = SNC; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp index ae63bf32b..e2fa8ca2e 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp @@ -29,7 +29,7 @@ REFLECT_END( CopyDirNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ CopyDirNode::CopyDirNode() -: Node( AString::GetEmpty(), Node::COPY_DIR_NODE, Node::FLAG_NONE ) + : Node( Node::COPY_DIR_NODE ) { } @@ -82,10 +82,8 @@ CopyDirNode::~CopyDirNode() = default; // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool CopyDirNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) +/*virtual*/ bool CopyDirNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { - (void)forceClean; // dynamic deps are always re-added here, so this is meaningless - m_DynamicDependencies.Clear(); ASSERT( !m_StaticDependencies.IsEmpty() ); @@ -119,7 +117,7 @@ CopyDirNode::~CopyDirNode() = default; Node * srcFileNode = nodeGraph.FindNode( srcFile ); if ( srcFileNode == nullptr ) { - srcFileNode = nodeGraph.CreateFileNode( srcFile ); + srcFileNode = nodeGraph.CreateNode( srcFile ); } else if ( srcFileNode->IsAFile() == false ) { @@ -135,7 +133,7 @@ CopyDirNode::~CopyDirNode() = default; Node * n = nodeGraph.FindNode( dstFile ); if ( n == nullptr ) { - CopyFileNode * copyFileNode = nodeGraph.CreateCopyFileNode( dstFile ); + CopyFileNode * copyFileNode = nodeGraph.CreateNode( dstFile ); copyFileNode->m_Source = srcFileNode->GetName(); copyFileNode->m_PreBuildDependencyNames = preBuildDependencyNames; // inherit PreBuildDependencies const BFFToken * token = nullptr; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h index ce56ad904..96c752d65 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.h @@ -25,7 +25,7 @@ class CopyDirNode : public Node virtual bool IsAFile() const override; private: - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; // Exposed Properties diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyFileNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyFileNode.cpp index 9abddd8d4..cc9c65ef6 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyFileNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyFileNode.cpp @@ -25,7 +25,7 @@ REFLECT_END( CopyFileNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ CopyFileNode::CopyFileNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() { m_Type = Node::COPY_FILE_NODE; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Dependencies.cpp b/Code/Tools/FBuild/FBuildCore/Graph/Dependencies.cpp index d234f7f40..f291a11f8 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Dependencies.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/Dependencies.cpp @@ -56,7 +56,7 @@ void Dependencies::Load( NodeGraph & nodeGraph, ConstMemoryStream & stream ) { return; } - + SetCapacity( numDeps ); for ( uint32_t i=0; i & patterns, + const Array & excludePaths, + const Array & filesToExclude, + const Array & excludePatterns, + bool recurse ) + : GetFilesHelper( patterns ) + , m_ExcludePaths( excludePaths ) + , m_FilesToExclude( filesToExclude ) + , m_ExcludePatterns( excludePatterns ) + { + m_Recurse = recurse; + } + + //-------------------------------------------------------------------------- + virtual bool OnDirectory( const AString & path ) override + { + if ( m_Recurse == false ) + { + return false; + } + + // Filter excluded paths + for ( const AString & excludedPath : m_ExcludePaths ) + { + if ( PathUtils::PathBeginsWith( path, excludedPath ) ) + { + return false; // Don't recurse into dir + } + } + + return true; // Recurse into directory + } + + //-------------------------------------------------------------------------- + virtual void OnFile( FileIO::FileInfo && info ) override + { + // filter excluded files + for ( const AString & fileToExclude : m_FilesToExclude ) + { + if ( PathUtils::PathEndsWithFile( info.m_Name, fileToExclude ) ) + { + return; // Exclude + } + } + + // Filter excluded patterns + for ( const AString & excludePattern : m_ExcludePatterns ) + { + if ( PathUtils::IsWildcardMatch( excludePattern.Get(), info.m_Name.Get() ) ) + { + return; // Exclude + } + } + + // Keep file info + m_Files.EmplaceBack( Move( info ) ); + } + + DirectoryListNodeGetFilesHelper& operator =(DirectoryListNodeGetFilesHelper&) = delete; +protected: + const Array & m_ExcludePaths; + const Array & m_FilesToExclude; + const Array & m_ExcludePatterns; +}; + // CONSTRUCTOR //------------------------------------------------------------------------------ DirectoryListNode::DirectoryListNode() - : Node( AString::GetEmpty(), Node::DIRECTORY_LIST_NODE, Node::FLAG_ALWAYS_BUILD ) + : Node( Node::DIRECTORY_LIST_NODE ) , m_Recursive( true ) , m_IncludeReadOnlyStatusInHash( false ) { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 100; } @@ -141,65 +214,20 @@ DirectoryListNode::~DirectoryListNode() = default; // NOTE: The DirectoryListNode makes no assumptions about whether no files // is an error or not. That's up to the dependent nodes to decide. - Array< FileIO::FileInfo > files( 4096, true ); - FileIO::GetFilesEx( m_Path, &m_Patterns, m_Recursive, &files ); - - m_Files.SetCapacity( files.GetSize() ); - - // filter exclusions - const FileIO::FileInfo * const end = files.End(); - for ( const FileIO::FileInfo * it = files.Begin(); it != end; it++ ) { - bool excluded = false; + // Get the list of files, filtered in various ways + DirectoryListNodeGetFilesHelper helper( m_Patterns, + m_ExcludePaths, + m_FilesToExclude, + m_ExcludePatterns, + m_Recursive ); + FileIO::GetFiles( m_Path, helper ); - // filter excluded paths - const AString * const eEnd = m_ExcludePaths.End(); - for ( const AString * eIt=m_ExcludePaths.Begin(); eIt != eEnd; ++eIt ) - { - if ( PathUtils::PathBeginsWith( it->m_Name, *eIt ) ) - { - excluded = true; - break; - } - } - - // filter excluded files - if ( !excluded ) - { - const AString * fit = m_FilesToExclude.Begin(); - const AString * const fend = m_FilesToExclude.End(); - for ( ; fit != fend; ++fit ) - { - if ( PathUtils::PathEndsWithFile( it->m_Name, *fit ) ) - { - excluded = true; - break; - } - } - } - - // filter excluded patterns - if ( !excluded ) - { - const AString * pit = m_ExcludePatterns.Begin(); - const AString * const pend = m_ExcludePatterns.End(); - for ( ; pit != pend; ++pit ) - { - if ( PathUtils::IsWildcardMatch( pit->Get(), it->m_Name.Get() ) ) - { - excluded = true; - break; - } - } - } - - if ( !excluded ) - { - m_Files.Append( *it ); - } + // Transfer ownership of filtered list + m_Files = Move( helper.GetFiles() ); } - MakePrettyName( files.GetSize() ); + MakePrettyName(); if ( FLog::ShowVerbose() ) { @@ -242,7 +270,7 @@ DirectoryListNode::~DirectoryListNode() = default; // MakePrettyName //------------------------------------------------------------------------------ -void DirectoryListNode::MakePrettyName( const size_t totalFiles ) +void DirectoryListNode::MakePrettyName() { AStackString<> prettyName( m_Path ); if (m_Recursive) @@ -251,7 +279,7 @@ void DirectoryListNode::MakePrettyName( const size_t totalFiles ) } const size_t numFiles = m_Files.GetSize(); - prettyName.AppendFormat( ", files kept: %zu / %zu", numFiles, totalFiles ); + prettyName.AppendFormat( ", files: %zu ", numFiles ); m_PrettyName = prettyName; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h index 0cb0cdf3e..30899b860 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h @@ -40,7 +40,7 @@ class DirectoryListNode : public Node private: virtual BuildResult DoBuild( Job * job ) override; - void MakePrettyName( const size_t totalFiles ); + void MakePrettyName(); friend class CompilationDatabase; // For DoBuild - TODO:C This is not ideal diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp index ad0cef1ce..3388cd6e5 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp @@ -45,7 +45,7 @@ REFLECT_END( ExecNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ ExecNode::ExecNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_ExecReturnCode( 0 ) , m_ExecAlwaysShowOutput( false ) , m_ExecUseStdOutAsOutput( false ) @@ -121,7 +121,7 @@ ExecNode::~ExecNode() // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool ExecNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool /*forceClean*/ ) +/*virtual*/ bool ExecNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { // clear dynamic deps from previous passes m_DynamicDependencies.Clear(); @@ -145,7 +145,7 @@ ExecNode::~ExecNode() Node * sn = nodeGraph.FindNode( file.m_Name ); if ( sn == nullptr ) { - sn = nodeGraph.CreateFileNode( file.m_Name ); + sn = nodeGraph.CreateNode( file.m_Name ); } else if ( sn->IsAFile() == false ) { @@ -217,7 +217,7 @@ ExecNode::~ExecNode() return NODE_RESULT_FAILED; } const bool buildFailed = ( result != m_ExecReturnCode ); - + // Print output if appropriate if ( buildFailed || m_ExecAlwaysShowOutput || diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h index 249861869..d2dc14064 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.h @@ -23,7 +23,7 @@ class ExecNode : public FileNode static inline Node::Type GetTypeS() { return Node::EXEC_NODE; } private: - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual bool DetermineNeedToBuildStatic() const override; virtual BuildResult DoBuild( Job * job ) override; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/FileNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/FileNode.cpp index f03677017..b3bb03c1f 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/FileNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/FileNode.cpp @@ -16,15 +16,9 @@ // CONSTRUCTOR //------------------------------------------------------------------------------ -FileNode::FileNode( const AString & fileName, uint8_t controlFlags ) - : Node( fileName, Node::FILE_NODE, controlFlags ) +FileNode::FileNode() + : Node( Node::FILE_NODE ) { - ASSERT( fileName.EndsWith( "\\" ) == false ); - #if defined( __WINDOWS__ ) - ASSERT( ( fileName.FindLast( ':' ) == nullptr ) || - ( fileName.FindLast( ':' ) == ( fileName.Get() + 1 ) ) ); - #endif - m_LastBuildTimeMs = 1; // very little work required } @@ -44,6 +38,12 @@ FileNode::~FileNode() = default; //------------------------------------------------------------------------------ /*virtual*/ Node::BuildResult FileNode::DoBuild( Job * /*job*/ ) { + ASSERT( m_Name.EndsWith( "\\" ) == false ); + #if defined( __WINDOWS__ ) + ASSERT( ( m_Name.FindLast( ':' ) == nullptr ) || + ( m_Name.FindLast( ':' ) == ( m_Name.Get() + 1 ) ) ); + #endif + // NOTE: Not calling RecordStampFromBuiltFile as this is not a built file m_Stamp = FileIO::GetFileLastWriteTime( m_Name ); // Don't assert m_Stamp != 0 as input file might not exist diff --git a/Code/Tools/FBuild/FBuildCore/Graph/FileNode.h b/Code/Tools/FBuild/FBuildCore/Graph/FileNode.h index 2f7cfe3f4..6312ccaff 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/FileNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/FileNode.h @@ -11,7 +11,7 @@ class FileNode : public Node { public: - explicit FileNode( const AString & fileName, uint8_t controlFlags ); + FileNode(); virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * funcStartIter, const Function * function ) override; virtual ~FileNode() override; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp index 1f6bec7f2..0a5ccbe54 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp @@ -34,7 +34,7 @@ REFLECT_NODE_BEGIN( LibraryNode, ObjectListNode, MetaName( "LibrarianOutput" ) + REFLECT( m_LibrarianOutput, "LibrarianOutput", MetaFile() ) REFLECT_ARRAY( m_LibrarianAdditionalInputs, "LibrarianAdditionalInputs", MetaOptional() + MetaFile() + MetaAllowNonFile( Node::OBJECT_LIST_NODE ) ) REFLECT( m_LibrarianAllowResponseFile, "LibrarianAllowResponseFile", MetaOptional() ) - REFLECT( m_LibrarianForceResponseFile, "LibrarianForceResponseFile", MetaOptional() ) + REFLECT( m_LibrarianForceResponseFile, "LibrarianForceResponseFile", MetaOptional() ) REFLECT( m_NumLibrarianAdditionalInputs, "NumLibrarianAdditionalInputs", MetaHidden() ) REFLECT( m_LibrarianFlags, "LibrarianFlags", MetaHidden() ) @@ -121,9 +121,9 @@ LibraryNode::~LibraryNode() // GatherDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool LibraryNode::GatherDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) +/*virtual*/ bool LibraryNode::GatherDynamicDependencies( NodeGraph & nodeGraph ) { - if ( ObjectListNode::GatherDynamicDependencies( nodeGraph, forceClean ) == false ) + if ( ObjectListNode::GatherDynamicDependencies( nodeGraph ) == false ) { return false; // GatherDynamicDependencies will have emited an error } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h index 86eff991e..f33b164ba 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.h @@ -42,7 +42,7 @@ class LibraryNode : public ObjectListNode private: friend class FunctionLibrary; - virtual bool GatherDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool GatherDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; // internal helpers diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp index 28bd76f8b..656580a2f 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp @@ -56,7 +56,7 @@ REFLECT_END( LinkerNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ LinkerNode::LinkerNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_LinkerType( "auto" ) , m_LinkerAllowResponseFile( false ) , m_LinkerForceResponseFile( false ) @@ -366,7 +366,7 @@ LinkerNode::~LinkerNode() } // Show output if desired - const bool showCommandOutput = ( result != 0 ) || + const bool showCommandOutput = ( result != 0 ) || FBuild::Get().GetOptions().m_ShowCommandOutput; if ( showCommandOutput ) { @@ -1378,7 +1378,7 @@ void LinkerNode::GetImportLibName( const AString & args, AString & importLibName // see if the file exists on disk at this location if ( LinkerNodeFileExistsCache::Get().FileExists( potentialNodeNameClean ) ) { - node = nodeGraph.CreateFileNode( potentialNodeNameClean ); + node = nodeGraph.CreateNode( potentialNodeNameClean, iter ); libs.Add( node ); found = true; FLOG_VERBOSE( "Additional library '%s' assumed to be '%s'\n", lib.Get(), potentialNodeNameClean.Get() ); @@ -1519,7 +1519,7 @@ void LinkerNode::GetImportLibName( const AString & args, AString & importLibName // node not found - create a new FileNode, assuming we are // linking against an externally built library - node = nodeGraph.CreateFileNode( nodeName ); + node = nodeGraph.CreateNode( nodeName, iter ); nodes.Add( node ); return true; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.h b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.h index 28369689c..a2c9f0309 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.h @@ -103,7 +103,7 @@ class LinkerNode : public FileNode Array< AString > m_LinkerAssemblyResources; bool m_LinkerLinkObjects = false; bool m_LinkerAllowResponseFile; - bool m_LinkerForceResponseFile; + bool m_LinkerForceResponseFile; AString m_LinkerStampExe; AString m_LinkerStampExeArgs; Array< AString > m_PreBuildDependencyNames; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ListDependenciesNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ListDependenciesNode.cpp index 09892edc1..f21b9df48 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ListDependenciesNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ListDependenciesNode.cpp @@ -87,7 +87,7 @@ class DependencyAscendingCompareIDeref // CONSTRUCTOR //------------------------------------------------------------------------------ ListDependenciesNode::ListDependenciesNode() -: FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() { m_Type = Node::LIST_DEPENDENCIES_NODE; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp index 8586bec10..caee10dc1 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.cpp @@ -47,7 +47,7 @@ #include "Core/FileIO/FileIO.h" #include "Core/FileIO/IOStream.h" #include "Core/FileIO/PathUtils.h" -#include "Core/Math/CRC32.h" +#include "Core/Math/xxHash.h" #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" #include "Core/Profile/Profile.h" @@ -122,12 +122,9 @@ REFLECT_END( Node ) // CONSTRUCTOR //------------------------------------------------------------------------------ -Node::Node( const AString & name, Type type, uint8_t controlFlags ) +Node::Node( Type type ) { m_Type = type; - m_ControlFlags = controlFlags; - - SetName( name ); // Compile time check to ensure name vector is in sync static_assert( sizeof( s_NodeTypeNames ) / sizeof(const char *) == NUM_NODE_TYPES, "s_NodeTypeNames item count doesn't match NUM_NODE_TYPES" ); @@ -139,7 +136,7 @@ Node::~Node() = default; // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool Node::DoDynamicDependencies( NodeGraph &, bool ) +/*virtual*/ bool Node::DoDynamicDependencies( NodeGraph & /*nodeGraph*/ ) { return true; } @@ -327,46 +324,6 @@ void Node::SetLastBuildTime( uint32_t ms ) AtomicStoreRelaxed( &m_LastBuildTimeMs, ms ); } -// CreateNode -//------------------------------------------------------------------------------ -/*static*/ Node * Node::CreateNode( NodeGraph & nodeGraph, Node::Type nodeType, const AString & name ) -{ - switch ( nodeType ) - { - case Node::PROXY_NODE: ASSERT( false ); return nullptr; - case Node::COPY_FILE_NODE: return nodeGraph.CreateCopyFileNode( name ); - case Node::DIRECTORY_LIST_NODE: return nodeGraph.CreateDirectoryListNode( name ); - case Node::EXEC_NODE: return nodeGraph.CreateExecNode( name ); - case Node::FILE_NODE: return nodeGraph.CreateFileNode( name ); - case Node::LIBRARY_NODE: return nodeGraph.CreateLibraryNode( name ); - case Node::OBJECT_NODE: return nodeGraph.CreateObjectNode( name ); - case Node::ALIAS_NODE: return nodeGraph.CreateAliasNode( name ); - case Node::EXE_NODE: return nodeGraph.CreateExeNode( name ); - case Node::CS_NODE: return nodeGraph.CreateCSNode( name ); - case Node::UNITY_NODE: return nodeGraph.CreateUnityNode( name ); - case Node::TEST_NODE: return nodeGraph.CreateTestNode( name ); - case Node::COMPILER_NODE: return nodeGraph.CreateCompilerNode( name ); - case Node::DLL_NODE: return nodeGraph.CreateDLLNode( name ); - case Node::VCXPROJECT_NODE: return nodeGraph.CreateVCXProjectNode( name ); - case Node::VSPROJEXTERNAL_NODE: return nodeGraph.CreateVSProjectExternalNode( name ); - case Node::OBJECT_LIST_NODE: return nodeGraph.CreateObjectListNode( name ); - case Node::COPY_DIR_NODE: return nodeGraph.CreateCopyDirNode( name ); - case Node::SLN_NODE: return nodeGraph.CreateSLNNode( name ); - case Node::REMOVE_DIR_NODE: return nodeGraph.CreateRemoveDirNode( name ); - case Node::XCODEPROJECT_NODE: return nodeGraph.CreateXCodeProjectNode( name ); - case Node::SETTINGS_NODE: return nodeGraph.CreateSettingsNode( name ); - case Node::TEXT_FILE_NODE: return nodeGraph.CreateTextFileNode( name ); - case Node::LIST_DEPENDENCIES_NODE: return nodeGraph.CreateListDependenciesNode( name ); - case Node::NUM_NODE_TYPES: ASSERT( false ); return nullptr; - } - - #if defined( __GNUC__ ) || defined( _MSC_VER ) - // GCC and incorrectly reports reaching end of non-void function (as of GCC 7.3.0) - // MSVC incorrectly reports reaching end of non-void function (as of VS 2017) - return nullptr; - #endif -} - // Load //------------------------------------------------------------------------------ /*static*/ Node * Node::Load( NodeGraph & nodeGraph, ConstMemoryStream & stream ) @@ -378,11 +335,11 @@ void Node::SetLastBuildTime( uint32_t ms ) PROFILE_SECTION( Node::GetTypeName( (Type)nodeType ) ); // Name of node - AStackString<> name; + AString name; VERIFY( stream.Read( name ) ); // Create node - Node * n = CreateNode( nodeGraph, (Type)nodeType, name ); + Node * n = nodeGraph.CreateNode( (Type)nodeType, Move( name ) ); ASSERT( n ); // Early out for FileNode @@ -398,7 +355,7 @@ void Node::SetLastBuildTime( uint32_t ms ) // Build time uint32_t lastTimeToBuild; VERIFY( stream.Read( lastTimeToBuild ) ); - n->SetLastBuildTime( lastTimeToBuild ); + n->SetLastBuildTime( lastTimeToBuild ); // Deserialize properties Deserialize( stream, n, *n->GetReflectionInfoV() ); @@ -756,10 +713,10 @@ void Node::SetLastBuildTime( uint32_t ms ) // SetName //------------------------------------------------------------------------------ -void Node::SetName( const AString & name ) +void Node::SetName( AString && name ) { - m_Name = name; - m_NameCRC = CRC32::CalcLower( name ); + m_NameHash = CalcNameHash( name ); + m_Name = Move( name ); } // ReplaceDummyName @@ -1088,19 +1045,29 @@ void Node::ReplaceDummyName( const AString & newName ) NodeGraph::CleanPath( path, outFixedPath ); } +// CalcNameHash +//------------------------------------------------------------------------------ +/*static*/ uint32_t Node::CalcNameHash( const AString & name ) +{ + // xxHash3 returns a 64 bit hash and we use the lower 32 bits + AStackString<> nameLower( name ); + nameLower.ToLower(); + return static_cast( xxHash3::Calc64( nameLower ) ); +} + // CleanMessageToPreventMSBuildFailure //------------------------------------------------------------------------------ /*static*/ void Node::CleanMessageToPreventMSBuildFailure( const AString & msg, AString & outMsg ) { // Search for patterns that MSBuild detects and treats as errors: - // + // // : - // + // // and remove the colon so they are no longer detected: - // + // // - // - // These can be anywhere in the string, and are case and whitespace insensitive + // + // These can be anywhere in the string, and are case and whitespace insensitive const char * pos = msg.Get(); for ( ;; ) { @@ -1178,18 +1145,18 @@ bool Node::InitializePreBuildDependencies( NodeGraph & nodeGraph, const BFFToken // No - return build-wide environment return FBuild::IsValid() ? FBuild::Get().GetEnvironmentString() : nullptr; } - + // More than one caller could be retrieving the same env string // in some cases. For simplicity, we protect in all cases even // if we could avoid it as the mutex will not be heavily constested. MutexHolder mh( g_NodeEnvStringMutex ); - + // If we've previously built a custom env string, use it if ( inoutCachedEnvString ) { return inoutCachedEnvString; } - + // Caller owns the memory inoutCachedEnvString = Env::AllocEnvironmentString( envVars ); return inoutCachedEnvString; @@ -1200,14 +1167,14 @@ bool Node::InitializePreBuildDependencies( NodeGraph & nodeGraph, const BFFToken void Node::RecordStampFromBuiltFile() { m_Stamp = FileIO::GetFileLastWriteTime( m_Name ); - + // An external tool might fail to write a file. Higher level code checks for // that (see "missing despite success"), so we don't need to do anything here. if ( m_Stamp == 0 ) { return; } - + // On OS X, the 'ar' tool (for making libraries) appears to clamp the // modification time of libraries to whole seconds. On HFS/HFS+ file systems, // this doesn't matter because the resolution of the file system is 1 second. @@ -1236,7 +1203,7 @@ void Node::RecordStampFromBuiltFile() { // Set to current time FileIO::SetFileLastWriteTimeToNow( m_Name ); - + // Re-query the time from the file m_Stamp = FileIO::GetFileLastWriteTime( m_Name ); ASSERT( m_Stamp != 0 ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/Node.h b/Code/Tools/FBuild/FBuildCore/Graph/Node.h index cb1deb0a3..b98e250d6 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/Node.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/Node.h @@ -114,20 +114,19 @@ class Node : public Struct enum State : uint8_t { - NOT_PROCESSED, // no work done (either not part of this build, or waiting on static dependencies ) - PRE_DEPS_READY, // pre-build deps processed - STATIC_DEPS_READY, // static dependencies are uptodate - we are ready to DoDynamicDeps - DYNAMIC_DEPS_DONE, // dynamic deps updated, waiting for dynamic deps to be ready + NOT_PROCESSED, // no work done (either not part of this build, or not yet seen) + STATIC_DEPS, // pre-build deps processed and checking static deps + DYNAMIC_DEPS, // dynamic deps regenerated and being checked BUILDING, // in the queue for building FAILED, // failed to build UP_TO_DATE, // built, or confirmed as not needing building }; - explicit Node( const AString & name, Type type, uint8_t controlFlags ); + explicit Node( Type type ); virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * funcStartIter, const Function * function ) = 0; virtual ~Node(); - inline uint32_t GetNameCRC() const { return m_NameCRC; } + uint32_t GetNameHash() const { return m_NameHash; } inline Type GetType() const { return m_Type; } inline const char * GetTypeName() const { return s_NodeTypeNames[ m_Type ]; } inline static const char * GetTypeName( Type t ) { return s_NodeTypeNames[ t ]; } @@ -150,7 +149,6 @@ class Node : public Struct inline uint32_t GetProgressAccumulator() const { return m_ProgressAccumulator; } inline void SetProgressAccumulator( uint32_t p ) const { m_ProgressAccumulator = p; } - static Node * CreateNode( NodeGraph & nodeGraph, Node::Type nodeType, const AString & name ); static Node * Load( NodeGraph & nodeGraph, ConstMemoryStream & stream ); static void LoadDependencies( NodeGraph & nodeGraph, Node * node, ConstMemoryStream & stream ); static void Save( IOStream & stream, const Node * node ); @@ -182,6 +180,8 @@ class Node : public Struct inline const Dependencies & GetStaticDependencies() const { return m_StaticDependencies; } inline const Dependencies & GetDynamicDependencies() const { return m_DynamicDependencies; } + static uint32_t CalcNameHash( const AString & name ); + static void CleanMessageToPreventMSBuildFailure( const AString & msg, AString & outMsg ); protected: @@ -197,7 +197,7 @@ class Node : public Struct friend class WorkerThread; friend class CompilationDatabase; - void SetName( const AString & name ); + void SetName( AString && name ); void ReplaceDummyName( const AString & newName ); @@ -210,7 +210,7 @@ class Node : public Struct // each node implements a subset of these as needed virtual bool DetermineNeedToBuildStatic() const; virtual bool DetermineNeedToBuildDynamic() const; - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ); + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ); virtual BuildResult DoBuild( Job * job ); virtual BuildResult DoBuild2( Job * job, bool racingRemoteJob ); virtual bool Finalize( NodeGraph & nodeGraph ); @@ -252,12 +252,12 @@ class Node : public Struct mutable uint16_t m_StatsFlags = 0; // Stats recorded in the current build mutable uint32_t m_BuildPassTag = 0; // Prevent multiple recursions into the same node during a single sweep uint64_t m_Stamp = 0; // "Stamp" representing this node for dependency comparissons - uint8_t m_ControlFlags; // Control build behavior special cases - Set by constructor + uint8_t m_ControlFlags = FLAG_NONE; // Control build behavior special cases - Set by constructor bool m_Hidden = false; // Hidden from -showtargets? // Note: Unused 2 bytes here uint32_t m_RecursiveCost = 0; // Recursive cost used during task ordering Node * m_Next = nullptr; // Node map in-place linked list pointer - uint32_t m_NameCRC; // Hash of mName. **Set by constructor** + uint32_t m_NameHash; // Hash of mName uint32_t m_LastBuildTimeMs = 0; // Time it took to do last known full build of this node uint32_t m_ProcessingTime = 0; // Time spent on this node during this build uint32_t m_CachingTime = 0; // Time spent caching this node diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp index fe46dcf36..822924ffc 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.cpp @@ -45,7 +45,6 @@ #include "Core/FileIO/FileStream.h" #include "Core/FileIO/MemoryStream.h" #include "Core/FileIO/PathUtils.h" -#include "Core/Math/CRC32.h" #include "Core/Math/xxHash.h" #include "Core/Mem/Mem.h" #include "Core/Process/Thread.h" @@ -159,7 +158,7 @@ NodeGraph::~NodeGraph() corruptDBName += ".corrupt"; FileIO::FileMove( AStackString<>( nodeGraphDBFile ), corruptDBName ); // Will overwrite if needed } - + // Create a fresh DB by parsing the BFF FDELETE( oldNG ); NodeGraph * newNG = FNEW( NodeGraph ); @@ -213,7 +212,8 @@ bool NodeGraph::ParseFromRoot( const char * bffFile ) // default instance if needed. const AStackString<> settingsNodeName( "$$Settings$$" ); const Node * settingsNode = FindNode( settingsNodeName ); - m_Settings = settingsNode ? settingsNode->CastTo< SettingsNode >() : CreateSettingsNode( settingsNodeName ); // Create a default + m_Settings = settingsNode ? settingsNode->CastTo() + : CreateNode( settingsNodeName, &BFFToken::GetBuiltInToken() ); // Create a default // Parser will populate m_UsedFiles const Array & usedFiles = bffParser.GetUsedFiles(); @@ -223,6 +223,11 @@ bool NodeGraph::ParseFromRoot( const char * bffFile ) m_UsedFiles.EmplaceBack( file->GetFileName(), file->GetTimeStamp(), file->GetHash() ); } } + + // Free token tracking data we no longer need (and won't be valid when + // BFFParser falls out of scope) + m_NodeSourceTokens.Destruct(); + return ok; } @@ -786,311 +791,122 @@ size_t NodeGraph::GetNodeCount() const // RegisterNode //------------------------------------------------------------------------------ -void NodeGraph::RegisterNode( Node * node ) +void NodeGraph::RegisterNode( Node * node, const BFFToken * sourceToken ) { ASSERT( Thread::IsMainThread() ); ASSERT( node->GetName().IsEmpty() == false ); ASSERT( FindNode( node->GetName() ) == nullptr ); AddNode( node ); + RegisterSourceToken( node, sourceToken ); } -// CreateCopyFileNode +// RegisterSourceToken //------------------------------------------------------------------------------ -CopyFileNode * NodeGraph::CreateCopyFileNode( const AString & dstFileName ) +void NodeGraph::RegisterSourceToken( const Node * node, const BFFToken * sourceToken ) { - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( dstFileName ) ); + // Where available, record the source token for the node + if ( sourceToken ) + { + // Ammortize array growth in parallel with m_AllNodes + m_NodeSourceTokens.SetCapacity( m_AllNodes.GetCapacity() ); - CopyFileNode * node = FNEW( CopyFileNode() ); - node->SetName( dstFileName ); - AddNode( node ); - return node; -} + // Nobody should have added this before + ASSERT( m_NodeSourceTokens.GetSize() < m_AllNodes.GetSize() ); -// CreateCopyDirNode -//------------------------------------------------------------------------------ -CopyDirNode * NodeGraph::CreateCopyDirNode( const AString & nodeName ) -{ - ASSERT( Thread::IsMainThread() ); + // Array may be non-contiguous, so fill it in + while ( m_NodeSourceTokens.GetSize() < m_AllNodes.GetSize() ) + { + m_NodeSourceTokens.EmplaceBack( nullptr ); + } - CopyDirNode * node = FNEW( CopyDirNode() ); - node->SetName( nodeName ); - AddNode( node ); - return node; + // Store the token in the parallel at the same place as the node + ASSERT( m_AllNodes.Top() == node ); (void)node; + m_NodeSourceTokens.Top() = sourceToken; + } } -// CreateRemoveDirNode +// CreateNode //------------------------------------------------------------------------------ -RemoveDirNode * NodeGraph::CreateRemoveDirNode( const AString & nodeName ) +Node * NodeGraph::CreateNode( Node::Type type, AString && name ) { ASSERT( Thread::IsMainThread() ); - RemoveDirNode * node = FNEW( RemoveDirNode() ); - node->SetName( nodeName ); + Node * node = nullptr; + switch ( type ) + { + case Node::PROXY_NODE: ASSERT( false ); return nullptr; + case Node::COPY_FILE_NODE: node = FNEW( CopyFileNode() ); break; + case Node::DIRECTORY_LIST_NODE: node = FNEW( DirectoryListNode() ); break; + case Node::EXEC_NODE: node = FNEW( ExecNode() ); break; + case Node::FILE_NODE: + { + node = FNEW( FileNode() ); + node->m_ControlFlags = Node::FLAG_ALWAYS_BUILD; // TODO:C Eliminate special case + break; + } + case Node::LIBRARY_NODE: node = FNEW( LibraryNode() ); break; + case Node::OBJECT_NODE: node = FNEW( ObjectNode() ); break; + case Node::ALIAS_NODE: node = FNEW( AliasNode() ); break; + case Node::EXE_NODE: node = FNEW( ExeNode() ); break; + case Node::CS_NODE: node = FNEW( CSNode() ); break; + case Node::UNITY_NODE: node = FNEW( UnityNode() ); break; + case Node::TEST_NODE: node = FNEW( TestNode() ); break; + case Node::COMPILER_NODE: node = FNEW( CompilerNode() ); break; + case Node::DLL_NODE: node = FNEW( DLLNode() ); break; + case Node::VCXPROJECT_NODE: node = FNEW( VCXProjectNode() ); break; + case Node::VSPROJEXTERNAL_NODE: node = FNEW( VSProjectExternalNode() ); break; + case Node::OBJECT_LIST_NODE: node = FNEW( ObjectListNode() ); break; + case Node::COPY_DIR_NODE: node = FNEW( CopyDirNode() ); break; + case Node::SLN_NODE: node = FNEW( SLNNode() ); break; + case Node::REMOVE_DIR_NODE: node = FNEW( RemoveDirNode() ); break; + case Node::XCODEPROJECT_NODE: node = FNEW( XCodeProjectNode() ); break; + case Node::SETTINGS_NODE: node = FNEW( SettingsNode() ); break; + case Node::TEXT_FILE_NODE: node = FNEW( TextFileNode() ); break; + case Node::LIST_DEPENDENCIES_NODE: node = FNEW( ListDependenciesNode() ); break; + case Node::NUM_NODE_TYPES: ASSERT( false ); return nullptr; + } + + ASSERT( node ); // All cases handled above means this is impossible + + // Names for files must be normalized by the time we get here + ASSERT( !node->IsAFile() || IsCleanPath( name ) ); + + // Store name and track new node + node->SetName( Move( name ) ); AddNode( node ); - return node; -} -// CreateExecNode -//------------------------------------------------------------------------------ -ExecNode * NodeGraph::CreateExecNode( const AString & nodeName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( nodeName ) ); - - ExecNode * node = FNEW( ExecNode() ); - node->SetName( nodeName ); - AddNode( node ); return node; } -// CreateFileNode + +// CreateNode //------------------------------------------------------------------------------ -FileNode * NodeGraph::CreateFileNode( const AString & fileName, bool cleanPath ) +Node * NodeGraph::CreateNode( Node::Type type, const AString & name, const BFFToken * sourceToken ) { - ASSERT( Thread::IsMainThread() ); + // Where possible callers should call the move version to transfer ownership + // of strings, but calers don't always have a string to transfer so this + // helper can be called in those situations + AString nameCopy; - FileNode * node; - - if ( cleanPath ) + // TODO:C Eliminate special case handling of FileNode + // For historical reasons users of FileNodes don't clean paths so we have to + // do it here. Ideally this special case would be eliminated in the future. + if ( type == Node::FILE_NODE ) { - AStackString< 512 > fullPath; - CleanPath( fileName, fullPath ); - node = FNEW( FileNode( fullPath, Node::FLAG_ALWAYS_BUILD ) ); + // Clean path + AStackString< 512 > cleanPath; + CleanPath( name, cleanPath ); + nameCopy = cleanPath; } else { - node = FNEW( FileNode( fileName, Node::FLAG_ALWAYS_BUILD ) ); + nameCopy = name; } - AddNode( node ); - return node; -} - -// CreateDirectoryListNode -//------------------------------------------------------------------------------ -DirectoryListNode * NodeGraph::CreateDirectoryListNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - - DirectoryListNode * node = FNEW( DirectoryListNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateLibraryNode -//------------------------------------------------------------------------------ -LibraryNode * NodeGraph::CreateLibraryNode( const AString & libraryName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( libraryName ) ); - - LibraryNode * node = FNEW( LibraryNode() ); - node->SetName( libraryName ); - AddNode( node ); - return node; -} - -// CreateObjectNode -//------------------------------------------------------------------------------ -ObjectNode * NodeGraph::CreateObjectNode( const AString & objectName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( objectName ) ); - - ObjectNode * node = FNEW( ObjectNode() ); - node->SetName( objectName ); - AddNode( node ); - return node; -} - -// CreateAliasNode -//------------------------------------------------------------------------------ -AliasNode * NodeGraph::CreateAliasNode( const AString & aliasName ) -{ - ASSERT( Thread::IsMainThread() ); - - AliasNode * node = FNEW( AliasNode() ); - node->SetName( aliasName ); - AddNode( node ); - return node; -} - -// CreateDLLNode -//------------------------------------------------------------------------------ -DLLNode * NodeGraph::CreateDLLNode( const AString & dllName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( dllName ) ); - - DLLNode * node = FNEW( DLLNode() ); - node->SetName( dllName ); - AddNode( node ); - return node; -} - -// CreateExeNode -//------------------------------------------------------------------------------ -ExeNode * NodeGraph::CreateExeNode( const AString & exeName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( exeName ) ); - - ExeNode * node = FNEW( ExeNode() ); - node->SetName( exeName ); - AddNode( node ); - return node; -} - -// CreateUnityNode -//------------------------------------------------------------------------------ -UnityNode * NodeGraph::CreateUnityNode( const AString & unityName ) -{ - ASSERT( Thread::IsMainThread() ); - - UnityNode * node = FNEW( UnityNode() ); - node->SetName( unityName ); - AddNode( node ); - return node; -} - -// CreateCSNode -//------------------------------------------------------------------------------ -CSNode * NodeGraph::CreateCSNode( const AString & csAssemblyName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( csAssemblyName ) ); - - CSNode * node = FNEW( CSNode() ); - node->SetName( csAssemblyName ); - AddNode( node ); - return node; -} - -// CreateTestNode -//------------------------------------------------------------------------------ -TestNode * NodeGraph::CreateTestNode( const AString & testOutput ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( testOutput ) ); - - TestNode * node = FNEW( TestNode() ); - node->SetName( testOutput ); - AddNode( node ); - return node; -} - -// CreateCompilerNode -//------------------------------------------------------------------------------ -CompilerNode * NodeGraph::CreateCompilerNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - - CompilerNode * node = FNEW( CompilerNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateVCXProjectNode -//------------------------------------------------------------------------------ -VSProjectBaseNode * NodeGraph::CreateVCXProjectNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( name ) ); - - VCXProjectNode * node = FNEW( VCXProjectNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateVSProjectExternalNode -//------------------------------------------------------------------------------ -VSProjectBaseNode * NodeGraph::CreateVSProjectExternalNode(const AString& name) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( name ) ); - - VSProjectExternalNode* node = FNEW( VSProjectExternalNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateSLNNode -//------------------------------------------------------------------------------ -SLNNode * NodeGraph::CreateSLNNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( name ) ); - - SLNNode * node = FNEW( SLNNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateObjectListNode -//------------------------------------------------------------------------------ -ObjectListNode * NodeGraph::CreateObjectListNode( const AString & listName ) -{ - ASSERT( Thread::IsMainThread() ); - - ObjectListNode * node = FNEW( ObjectListNode() ); - node->SetName( listName ); - AddNode( node ); - return node; -} - -// CreateXCodeProjectNode -//------------------------------------------------------------------------------ -XCodeProjectNode * NodeGraph::CreateXCodeProjectNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( name ) ); + Node * node = CreateNode( type, Move( nameCopy ) ); - XCodeProjectNode * node = FNEW( XCodeProjectNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} + RegisterSourceToken( node, sourceToken ); -// CreateSettingsNode -//------------------------------------------------------------------------------ -SettingsNode * NodeGraph::CreateSettingsNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - - SettingsNode * node = FNEW( SettingsNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateListDependenciesNode -//------------------------------------------------------------------------------ -ListDependenciesNode * NodeGraph::CreateListDependenciesNode( const AString & name ) -{ - ASSERT( Thread::IsMainThread() ); - - ListDependenciesNode * node = FNEW( ListDependenciesNode() ); - node->SetName( name ); - AddNode( node ); - return node; -} - -// CreateTextFileNode -//------------------------------------------------------------------------------ -TextFileNode* NodeGraph::CreateTextFileNode( const AString& nodeName ) -{ - ASSERT( Thread::IsMainThread() ); - ASSERT( IsCleanPath( nodeName ) ); - - TextFileNode* node = FNEW( TextFileNode() ); - node->SetName( nodeName ); - AddNode( node ); return node; } @@ -1105,7 +921,7 @@ void NodeGraph::AddNode( Node * node ) ASSERT( FindNodeInternal( node->GetName() ) == nullptr ); // node name must be unique // track in NodeMap - const uint32_t crc = CRC32::CalcLower( node->GetName() ); + const uint32_t crc = Node::CalcNameHash( node->GetName() ); const size_t key = ( crc & m_NodeMapMaxKey ); node->m_Next = m_NodeMap[ key ]; m_NodeMap[ key ] = node; @@ -1177,108 +993,108 @@ void NodeGraph::BuildRecurse( Node * nodeToBuild, uint32_t cost ) { ASSERT( nodeToBuild ); - // already building, or queued to build? - ASSERT( nodeToBuild->GetState() != Node::BUILDING ); - // accumulate recursive cost cost += nodeToBuild->GetLastBuildTime(); - // check pre-build dependencies - if ( nodeToBuild->GetState() == Node::NOT_PROCESSED ) + // False positive "Unannotated fallthrough between switch labels" (VS 2019 v14.29.30037) + #if ( _MSC_VER < 1935 ) + PRAGMA_DISABLE_PUSH_MSVC(26819) + #endif + + switch ( nodeToBuild->GetState() ) { - // all pre-build deps done? - const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetPreBuildDependencies(), cost ); - if ( allDependenciesUpToDate == false ) + case Node::NOT_PROCESSED: { - return; // not ready or failed - } + // check pre-build dependencies + const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetPreBuildDependencies(), cost ); + if ( allDependenciesUpToDate == false ) + { + return; // not ready or failed + } + nodeToBuild->SetState( Node::STATIC_DEPS ); - nodeToBuild->SetState( Node::PRE_DEPS_READY ); - } + [[fallthrough]]; + } + case Node::STATIC_DEPS: + { + // check static dependencies + const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetStaticDependencies(), cost ); + if ( allDependenciesUpToDate == false ) + { + return; // not ready or failed + } - ASSERT( ( nodeToBuild->GetState() == Node::PRE_DEPS_READY ) || - ( nodeToBuild->GetState() == Node::STATIC_DEPS_READY ) || - ( nodeToBuild->GetState() == Node::DYNAMIC_DEPS_DONE ) ); + // If static deps require us to rebuild, dynamic dependencies need regenerating + if ( FBuild::Get().GetOptions().m_ForceCleanBuild || + nodeToBuild->DetermineNeedToBuildStatic() ) + { + // Explicitly mark node in a way that will result in it rebuilding should + // we cancel the build before builing this node + if ( nodeToBuild->m_Stamp == 0 ) + { + // Note that this is the first time we're building (since Node can't check + // stamp as we clear it below) + nodeToBuild->SetStatFlag( Node::STATS_FIRST_BUILD ); + } + nodeToBuild->m_Stamp = 0; - // test static dependencies first - if ( nodeToBuild->GetState() == Node::PRE_DEPS_READY ) - { - // all static deps done? - const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetStaticDependencies(), cost ); - if ( allDependenciesUpToDate == false ) - { - return; // not ready or failed - } + // Regenerate dynamic dependencies + nodeToBuild->m_DynamicDependencies.Clear(); + if ( nodeToBuild->DoDynamicDependencies( *this ) == false ) + { + nodeToBuild->SetState( Node::FAILED ); + return; + } - nodeToBuild->SetState( Node::STATIC_DEPS_READY ); - } + // Continue through to check dynamic dependencies and build + } - ASSERT( ( nodeToBuild->GetState() == Node::STATIC_DEPS_READY ) || - ( nodeToBuild->GetState() == Node::DYNAMIC_DEPS_DONE ) ); + // Dynamic dependencies are ready to be checked + nodeToBuild->SetState( Node::DYNAMIC_DEPS ); - if ( nodeToBuild->GetState() != Node::DYNAMIC_DEPS_DONE ) - { - // If static deps require us to rebuild, dynamic dependencies need regenerating - const bool forceClean = FBuild::Get().GetOptions().m_ForceCleanBuild; - if ( forceClean || - nodeToBuild->DetermineNeedToBuildStatic() ) + [[fallthrough]]; + } + case Node::DYNAMIC_DEPS: { - // Clear dynamic dependencies - nodeToBuild->m_DynamicDependencies.Clear(); - - // Explicitly mark node in a way that will result in it rebuilding should - // we cancel the build before builing this node - if ( nodeToBuild->m_Stamp == 0 ) + // check dynamic dependencies + const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetDynamicDependencies(), cost ); + if ( allDependenciesUpToDate == false ) { - // Note that this is the first time we're building (since Node can't check - // stamp as we clear it below) - nodeToBuild->SetStatFlag( Node::STATS_FIRST_BUILD ); + return; // not ready or failed } - nodeToBuild->m_Stamp = 0; - // Regenerate dynamic dependencies - if ( nodeToBuild->DoDynamicDependencies( *this, forceClean ) == false ) + // dependencies are uptodate, so node can now tell us if it needs + // building + nodeToBuild->SetStatFlag( Node::STATS_PROCESSED ); + if ( ( nodeToBuild->GetStamp() == 0 ) || // Avoid redundant work in DetermineNeedToBuild + nodeToBuild->DetermineNeedToBuildDynamic() ) { - nodeToBuild->SetState( Node::FAILED ); - return; + nodeToBuild->m_RecursiveCost = cost; + JobQueue::Get().AddJobToBatch( nodeToBuild ); } - - // Continue through to check dynamic dependencies and build + else + { + if ( FLog::ShowVerbose() ) + { + FLOG_BUILD_REASON( "Up-To-Date '%s'\n", nodeToBuild->GetName().Get() ); + } + nodeToBuild->SetState( Node::UP_TO_DATE ); + } + break; } - - // Dynamic dependencies are ready to be checked - nodeToBuild->SetState( Node::DYNAMIC_DEPS_DONE ); - } - - ASSERT( nodeToBuild->GetState() == Node::DYNAMIC_DEPS_DONE ); - - // dynamic deps - { - // all static deps done? - const bool allDependenciesUpToDate = CheckDependencies( nodeToBuild, nodeToBuild->GetDynamicDependencies(), cost ); - if ( allDependenciesUpToDate == false ) + case Node::BUILDING: + case Node::FAILED: + case Node::UP_TO_DATE: { - return; // not ready or failed + ASSERT(false); // Should be impossible + break; } } - // dependencies are uptodate, so node can now tell us if it needs - // building - nodeToBuild->SetStatFlag( Node::STATS_PROCESSED ); - if ( ( nodeToBuild->GetStamp() == 0 ) || // Avoid redundant messages from DetermineNeedToBuild - nodeToBuild->DetermineNeedToBuildDynamic() ) - { - nodeToBuild->m_RecursiveCost = cost; - JobQueue::Get().AddJobToBatch( nodeToBuild ); - } - else - { - if ( FLog::ShowVerbose() ) - { - FLOG_BUILD_REASON( "Up-To-Date '%s'\n", nodeToBuild->GetName().Get() ); - } - nodeToBuild->SetState( Node::UP_TO_DATE ); - } + // False positive "Unannotated fallthrough between switch labels" (VS 2019 v14.29.30037) + #if ( _MSC_VER < 1935 ) + PRAGMA_DISABLE_POP_MSVC // 26819 + #endif } // CheckDependencies @@ -1370,6 +1186,23 @@ void NodeGraph::SetBuildPassTagForAllNodes( uint32_t value ) const } } +// FindNodeSourceToken +//------------------------------------------------------------------------------ +const BFFToken * NodeGraph::FindNodeSourceToken( const Node * node ) const +{ + // Find the index of the node. This is a slow operation, but we only expect + // this to happen in non-critical paths like when emitting BFF parsing errors + const size_t index = m_AllNodes.GetIndexOf( m_AllNodes.Find( node ) ); + + // Return token if available. Not all nodes have creation info available. + if ( index < m_NodeSourceTokens.GetSize() ) + { + return m_NodeSourceTokens[ index ]; + } + + return nullptr; +} + // CleanPath //------------------------------------------------------------------------------ /*static*/ void NodeGraph::CleanPath( AString & name, bool makeFullPath ) @@ -1536,15 +1369,15 @@ Node * NodeGraph::FindNodeInternal( const AString & fullPath ) const { ASSERT( Thread::IsMainThread() ); - const uint32_t crc = CRC32::CalcLower( fullPath ); - const size_t key = ( crc & m_NodeMapMaxKey ); + const uint32_t hash = Node::CalcNameHash( fullPath ); + const size_t key = ( hash & m_NodeMapMaxKey ); Node * n = m_NodeMap[ key ]; while ( n ) { - if ( n->GetNameCRC() == crc ) + if ( n->GetNameHash() == hash ) { - if ( n->GetName().CompareI( fullPath ) == 0 ) + if ( n->GetName().EqualsI( fullPath ) ) { return n; } @@ -1725,8 +1558,8 @@ void NodeGraph::FindNearestNodesInternal( const AString & fullPath, Array< NodeW // // Some of these things depend on timing, so this check could conceivably run // when not stuck, but it will never falsly detect a cyclic dependency. - // - + // + // Early out if the root node is being processed if ( node->GetState() >= Node::State::BUILDING ) { @@ -1747,7 +1580,7 @@ void NodeGraph::FindNearestNodesInternal( const AString & fullPath, Array< NodeW JobQueue::Get().GetJobStats( numJobs, numJobsActive, numJobsDist, numJobsDistActive ); if ( ( numJobs > 0 ) || ( numJobsActive > 0 ) || - ( numJobsDist > 0 ) || + ( numJobsDist > 0 ) || ( numJobsDistActive > 0 ) ) { return false; @@ -2079,7 +1912,7 @@ void NodeGraph::MigrateNode( const NodeGraph & oldNodeGraph, Node & newNode, con else { // Create the dependency - newDepNode = Node::CreateNode( *this, oldDepNode->GetType(), oldDepNode->GetName() ); + newDepNode = CreateNode( oldDepNode->GetType(), oldDepNode->GetName() ); ASSERT( newDepNode ); newDeps.Add( newDepNode, oldDep.GetNodeStamp(), oldDep.IsWeak() ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index 59880707d..cee3065ba 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -7,6 +7,7 @@ #include "Tools/FBuild/FBuildCore/BFF/BFFFileExists.h" #include "Tools/FBuild/FBuildCore/Helpers/SLNGenerator.h" #include "Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.h" +#include "Tools/FBuild/FBuildCore/Graph/Node.h" #include "Core/Containers/Array.h" #include "Core/Strings/AString.h" @@ -63,7 +64,7 @@ class NodeGraphHeader } inline ~NodeGraphHeader() = default; - enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 170 }; + enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 171 }; bool IsValid() const; bool IsCompatibleVersion() const { return m_Version == NODE_GRAPH_CURRENT_VERSION; } @@ -109,38 +110,27 @@ class NodeGraph size_t GetNodeCount() const; const SettingsNode * GetSettings() const { return m_Settings; } - void RegisterNode( Node * n ); + void RegisterNode( Node * n, const BFFToken * sourceToken ); // create new nodes - CopyFileNode * CreateCopyFileNode( const AString & dstFileName ); - CopyDirNode * CreateCopyDirNode( const AString & nodeName ); - RemoveDirNode * CreateRemoveDirNode( const AString & nodeName ); - ExecNode * CreateExecNode( const AString & dstFileName ); - FileNode * CreateFileNode( const AString & fileName, bool cleanPath = true ); - DirectoryListNode * CreateDirectoryListNode( const AString & name ); - LibraryNode * CreateLibraryNode( const AString & libraryName ); - ObjectNode * CreateObjectNode( const AString & objectName ); - AliasNode * CreateAliasNode( const AString & aliasName ); - DLLNode * CreateDLLNode( const AString & dllName ); - ExeNode * CreateExeNode( const AString & exeName ); - UnityNode * CreateUnityNode( const AString & unityName ); - CSNode * CreateCSNode( const AString & csAssemblyName ); - TestNode * CreateTestNode( const AString & testOutput ); - CompilerNode * CreateCompilerNode( const AString & name ); - VSProjectBaseNode * CreateVCXProjectNode( const AString & name ); - VSProjectBaseNode * CreateVSProjectExternalNode( const AString& name ); - SLNNode * CreateSLNNode( const AString & name ); - ObjectListNode * CreateObjectListNode( const AString & listName ); - XCodeProjectNode * CreateXCodeProjectNode( const AString & name ); - SettingsNode * CreateSettingsNode( const AString & name ); - ListDependenciesNode* CreateListDependenciesNode( const AString& name ); - TextFileNode * CreateTextFileNode( const AString & name ); + Node * CreateNode( Node::Type type, AString && name ); + Node * CreateNode( Node::Type type, + const AString & name, + const BFFToken * sourceToken = nullptr ); + template + T * CreateNode( const AString & name, + const BFFToken * sourceToken = nullptr ) + { + return CreateNode( T::GetTypeS(), name, sourceToken )->template CastTo(); + } void DoBuildPass( Node * nodeToBuild ); // Non-build operations that use the BuildPassTag can set it to a known value void SetBuildPassTagForAllNodes( uint32_t value ) const; + const BFFToken * FindNodeSourceToken( const Node * node ) const; + static void CleanPath( AString & name, bool makeFullPath = true ); static void CleanPath( const AString & name, AString & cleanPath, bool makeFullPath = true ); #if defined( ASSERTS_ENABLED ) @@ -190,6 +180,8 @@ class NodeGraph bool & movedDB ) const; uint32_t GetLibEnvVarHash() const; + void RegisterSourceToken( const Node * node, const BFFToken * sourceToken ); + // load/save helpers static void SerializeToText( Node * node, uint32_t depth, AString & outBuffer ); static void SerializeToText( const char * title, const Dependencies & dependencies, uint32_t depth, AString & outBuffer ); @@ -231,6 +223,8 @@ class NodeGraph }; Array< UsedFile > m_UsedFiles; + Array< const BFFToken * > m_NodeSourceTokens; + const SettingsNode * m_Settings; static uint32_t s_BuildPassTag; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.cpp b/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.cpp index ab6f13dda..b12900b55 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.cpp @@ -7,9 +7,10 @@ // CONSTRUCTOR //------------------------------------------------------------------------------ -NodeProxy::NodeProxy( const AString & name ) - : Node( name, Node::PROXY_NODE, 0 ) +NodeProxy::NodeProxy( AString && name ) + : Node( Node::PROXY_NODE ) { + SetName( Move( name ) ); } // DESTRUCTOR diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.h index 3c05f96c2..c33c0197a 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeProxy.h @@ -11,7 +11,7 @@ class NodeProxy : public Node { public: - explicit NodeProxy( const AString & name ); + explicit NodeProxy( AString && name ); virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * funcStartIter, const Function * function ) override; virtual ~NodeProxy() override; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index e5649bcf5..9c089bb8f 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -77,7 +77,7 @@ REFLECT_END( ObjectListNode ) // ObjectListNode //------------------------------------------------------------------------------ ObjectListNode::ObjectListNode() -: Node( AString::GetEmpty(), Node::OBJECT_LIST_NODE, Node::FLAG_NONE ) + : Node( Node::OBJECT_LIST_NODE ) { m_LastBuildTimeMs = 10000; @@ -344,10 +344,8 @@ ObjectListNode::~ObjectListNode() = default; // GatherDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool ObjectListNode::GatherDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) +/*virtual*/ bool ObjectListNode::GatherDynamicDependencies( NodeGraph & nodeGraph ) { - (void)forceClean; // dynamic deps are always re-added here, so this is meaningless - // clear dynamic deps from previous passes m_DynamicDependencies.Clear(); @@ -371,7 +369,7 @@ ObjectListNode::~ObjectListNode() = default; Node * n = nodeGraph.FindNode( fIt->m_Name ); if ( n == nullptr ) { - n = nodeGraph.CreateFileNode( fIt->m_Name ); + n = nodeGraph.CreateNode( fIt->m_Name ); } else if ( n->IsAFile() == false ) { @@ -409,7 +407,7 @@ ObjectListNode::~ObjectListNode() = default; Node * n = nodeGraph.FindNode( *it ); if ( n == nullptr ) { - n = nodeGraph.CreateFileNode( *it ); + n = nodeGraph.CreateNode( *it ); } else if ( n->IsAFile() == false ) { @@ -430,7 +428,7 @@ ObjectListNode::~ObjectListNode() = default; Node * n = nodeGraph.FindNode( isolatedFile.GetFileName() ); if ( n == nullptr ) { - n = nodeGraph.CreateFileNode( isolatedFile.GetFileName() ); + n = nodeGraph.CreateNode( isolatedFile.GetFileName() ); } else if ( n->IsAFile() == false ) { @@ -513,9 +511,9 @@ ObjectListNode::~ObjectListNode() = default; // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool ObjectListNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) +/*virtual*/ bool ObjectListNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { - if ( GatherDynamicDependencies( nodeGraph, forceClean ) == false ) + if ( GatherDynamicDependencies( nodeGraph ) == false ) { return false; // GatherDynamicDependencies will have emitted error } @@ -777,7 +775,7 @@ ObjectNode * ObjectListNode::CreateObjectNode( NodeGraph & nodeGraph, const AString & objectInput, const AString & pchObjectName ) { - ObjectNode * node= nodeGraph.CreateObjectNode( objectName ); + ObjectNode * node= nodeGraph.CreateNode( objectName, iter ); node->m_Compiler = m_Compiler; node->m_CompilerOptions = compilerOptions; node->m_CompilerOptionsDeoptimized = compilerOptionsDeoptimized; @@ -838,7 +836,7 @@ void ObjectListNode::EnumerateInputFiles( void (*callback)( const AString & inpu { callback( file, AString::GetEmpty(), userData ); } - + // Dynamically discovered files for ( size_t i = m_ObjectListInputStartIndex; i < m_ObjectListInputEndIndex; ++i ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h index afce005ea..feda55d52 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.h @@ -49,8 +49,8 @@ class ObjectListNode : public Node protected: friend class FunctionObjectList; - virtual bool GatherDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ); - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool GatherDynamicDependencies( NodeGraph & nodeGraph ); + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; // internal helpers diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp index 0b139f1df..c81c1b039 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp @@ -95,7 +95,7 @@ REFLECT_END( ObjectNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ ObjectNode::ObjectNode() -: FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() { m_Type = OBJECT_NODE; m_LastBuildTimeMs = 5000; // higher default than a file node @@ -170,14 +170,15 @@ ObjectNode::ObjectNode() // CONSTRUCTOR (Remote) //------------------------------------------------------------------------------ -ObjectNode::ObjectNode( const AString & objectName, +ObjectNode::ObjectNode( AString && objectName, NodeProxy * srcFile, const AString & compilerOptions, uint32_t flags ) -: FileNode( objectName, Node::FLAG_NONE ) -, m_CompilerOptions( compilerOptions ) -, m_Remote( true ) + : FileNode() + , m_CompilerOptions( compilerOptions ) + , m_Remote( true ) { + SetName( Move( objectName ) ); m_Type = OBJECT_NODE; m_LastBuildTimeMs = 5000; // higher default than a file node m_CompilerFlags.m_Flags = flags; @@ -294,7 +295,7 @@ ObjectNode::~ObjectNode() Node * fn = nodeGraph.FindNode( *it ); if ( fn == nullptr ) { - fn = nodeGraph.CreateFileNode( *it ); + fn = nodeGraph.CreateNode( *it ); } else if ( fn->IsAFile() == false ) { @@ -893,8 +894,8 @@ bool ObjectNode::ProcessIncludesWithPreProcessor( Job * job ) //------------------------------------------------------------------------------ /*static*/ Node * ObjectNode::LoadRemote( IOStream & stream ) { - AStackString<> name; - AStackString<> sourceFile; + AString name; + AString sourceFile; uint32_t flags; AStackString<> compilerArgs; if ( ( stream.Read( name ) == false ) || @@ -905,9 +906,9 @@ bool ObjectNode::ProcessIncludesWithPreProcessor( Job * job ) return nullptr; } - NodeProxy * srcFile = FNEW( NodeProxy( sourceFile ) ); + NodeProxy * srcFile = FNEW( NodeProxy( Move( sourceFile ) ) ); - return FNEW( ObjectNode( name, srcFile, compilerArgs, flags ) ); + return FNEW( ObjectNode( Move( name ), srcFile, compilerArgs, flags ) ); } // DetermineFlags @@ -1410,7 +1411,7 @@ bool ObjectNode::RetrieveFromCache( Job * job ) { pchKey = xxHash3::Calc64( cacheData, cacheDataSize ); } - + const uint32_t startDecompress = uint32_t( t.GetElapsedMS() ); MultiBuffer buffer( cacheData, cacheDataSize ); @@ -2329,13 +2330,13 @@ bool ObjectNode::CompileHelper::SpawnCompiler( Job * job, { Thread::Sleep( 1 ); } - + // Add fake failure ASSERT( m_Result == 0 ); // Should not have real failures if we're faking them m_Result = 1; job->Error( "Injecting system failure (sFakeSystemFailure)\n" ); job->OnSystemError(); - + // Clear failure state sFakeSystemFailureState.Store( DISABLED ); } @@ -2550,7 +2551,7 @@ bool ObjectNode::CompileHelper::SpawnCompiler( Job * job, } } - #if !defined( __WINDOWS__) + #if !defined( __WINDOWS__) (void)stdOut; // No checks use stdOut outside of Windows right now #endif } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.h b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.h index f8ce67c64..ce2784c42 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.h @@ -41,7 +41,7 @@ class ObjectNode : public FileNode ObjectNode(); virtual bool Initialize( NodeGraph & nodeGraph, const BFFToken * iter, const Function * function ) override; // simplified remote constructor - explicit ObjectNode( const AString & objectName, + explicit ObjectNode( AString && objectName, NodeProxy * srcFile, const AString & compilerOptions, uint32_t flags ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp index a2840b840..56905ab8d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp @@ -28,9 +28,10 @@ REFLECT_END( RemoveDirNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ RemoveDirNode::RemoveDirNode() - : Node( AString::GetEmpty(), Node::REMOVE_DIR_NODE, Node::FLAG_ALWAYS_BUILD ) + : Node( Node::REMOVE_DIR_NODE ) , m_RemovePathsRecurse( true ) { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_RemovePatterns.EmplaceBack( "*" ); } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp index b6a85ba5d..8ab8070b8 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/SLNNode.cpp @@ -76,8 +76,9 @@ struct VCXProjectNodeComp // CONSTRUCTOR //------------------------------------------------------------------------------ SLNNode::SLNNode() - : FileNode( AString::GetEmpty(), Node::FLAG_ALWAYS_BUILD ) + : FileNode() { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 100; // higher default than a file node m_Type = Node::SLN_NODE; } @@ -163,7 +164,7 @@ SLNNode::SLNNode() { // Merge list of projects found->m_Projects.Append( folder.m_Projects ); - + // Merge list of items found->m_Items.Append( folder.m_Items ); } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/SettingsNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/SettingsNode.cpp index 6f33cca20..c0bdba488 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/SettingsNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/SettingsNode.cpp @@ -37,9 +37,9 @@ REFLECT_END( SettingsNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ SettingsNode::SettingsNode() -: Node( AString::GetEmpty(), Node::SETTINGS_NODE, Node::FLAG_NONE ) -, m_WorkerConnectionLimit( 15 ) -, m_DistributableJobMemoryLimitMiB( DIST_MEMORY_LIMIT_DEFAULT ) + : Node( Node::SETTINGS_NODE ) + , m_WorkerConnectionLimit( 15 ) + , m_DistributableJobMemoryLimitMiB( DIST_MEMORY_LIMIT_DEFAULT ) { // Cache path from environment Env::GetEnvVariable( "FASTBUILD_CACHE_PATH", m_CachePathFromEnvVar ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp index 696da25f7..947c9b90d 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.cpp @@ -43,7 +43,7 @@ REFLECT_END( TestNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ TestNode::TestNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_TestExecutable() , m_TestArguments() , m_TestWorkingDir() @@ -126,7 +126,7 @@ const char * TestNode::GetEnvironmentString() const // DoDynamicDependencies //------------------------------------------------------------------------------ -/*virtual*/ bool TestNode::DoDynamicDependencies( NodeGraph & nodeGraph, bool /*forceClean*/ ) +/*virtual*/ bool TestNode::DoDynamicDependencies( NodeGraph & nodeGraph ) { // clear dynamic deps from previous passes m_DynamicDependencies.Clear(); @@ -150,7 +150,7 @@ const char * TestNode::GetEnvironmentString() const Node * sn = nodeGraph.FindNode( file.m_Name ); if ( sn == nullptr ) { - sn = nodeGraph.CreateFileNode( file.m_Name ); + sn = nodeGraph.CreateNode( file.m_Name ); } else if ( sn->IsAFile() == false ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.h b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.h index fd497adca..932bde804 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/TestNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/TestNode.h @@ -26,7 +26,7 @@ class TestNode : public FileNode const char * GetEnvironmentString() const; private: - virtual bool DoDynamicDependencies( NodeGraph & nodeGraph, bool forceClean ) override; + virtual bool DoDynamicDependencies( NodeGraph & nodeGraph ) override; virtual BuildResult DoBuild( Job * job ) override; void EmitCompilationMessage( const char * workingDir ) const; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp index ba5a579a8..25fec934b 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/TextFileNode.cpp @@ -30,7 +30,7 @@ REFLECT_END( TextFileNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ TextFileNode::TextFileNode() - : FileNode( AString::GetEmpty(), Node::FLAG_NONE ) + : FileNode() , m_TextFileAlways( false ) { m_Type = TEXT_FILE_NODE; @@ -57,7 +57,7 @@ TextFileNode::~TextFileNode() = default; // DetermineNeedToBuildStatic //------------------------------------------------------------------------------ /*virtual*/ bool TextFileNode::DetermineNeedToBuildStatic() const -{ +{ if ( m_TextFileAlways ) { FLOG_VERBOSE( "Need to build '%s' (TextFileAlways = true)", GetName().Get() ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp index fc52a1ec4..6f9953ae1 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp @@ -142,7 +142,7 @@ bool UnityNode::UnityFileAndOrigin::operator < ( const UnityFileAndOrigin & othe // CONSTRUCTOR //------------------------------------------------------------------------------ UnityNode::UnityNode() - : Node( AString::GetEmpty(), Node::UNITY_NODE, Node::FLAG_NONE ) + : Node( Node::UNITY_NODE ) , m_InputPathRecurse( true ) , m_InputPattern( 1, true ) , m_Files( 0, true ) @@ -806,7 +806,7 @@ bool UnityNode::GetIsolatedFilesFromList( Array< AString > & files ) const { return true; // No list specified so option is disabled } - + // Open file FileStream input; if ( input.Open( m_IsolateListFile.Get() ) == false ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp index 1c8afe2d4..7dd2eb9d6 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp @@ -69,6 +69,7 @@ REFLECT_STRUCT_BEGIN_BASE( VSProjectConfigBase ) REFLECT( m_LinuxProjectType, "LinuxProjectType", MetaInheritFromOwner() + MetaOptional() ) REFLECT( m_PackagePath, "PackagePath", MetaInheritFromOwner() + MetaOptional() ) REFLECT( m_AdditionalSymbolSearchPaths, "AdditionalSymbolSearchPaths", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidApkLocation, "AndroidApkLocation", MetaInheritFromOwner() + MetaOptional() ) REFLECT_END( VSProjectConfigBase ) REFLECT_STRUCT_BEGIN( VSProjectConfig, VSProjectConfigBase, MetaNone() ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h index 82e44630f..f63301957 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h @@ -59,6 +59,7 @@ class VSProjectConfigBase : public Struct AString m_LinuxProjectType; AString m_PackagePath; AString m_AdditionalSymbolSearchPaths; + AString m_AndroidApkLocation; }; // VSProjectConfig diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VSProjectBaseNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VSProjectBaseNode.cpp index 8e60fd936..eeb32ad99 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VSProjectBaseNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VSProjectBaseNode.cpp @@ -14,8 +14,9 @@ REFLECT_END( VSProjectBaseNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ VSProjectBaseNode::VSProjectBaseNode() - : FileNode( AString::GetEmpty(), Node::FLAG_ALWAYS_BUILD ) + : FileNode() { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 100; // higher default than a file node } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VSProjectExternalNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VSProjectExternalNode.cpp index aa58049b4..1b03ade8c 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VSProjectExternalNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VSProjectExternalNode.cpp @@ -16,7 +16,7 @@ #if defined( __WINDOWS__ ) #include "Core/Env/WindowsHeader.h" PRAGMA_DISABLE_PUSH_CLANG( "-Wunknown-warning-option" ) - PRAGMA_DISABLE_PUSH_CLANG( "-Wreserved-identifier" ) // identifier '%s' is reserved because it starts with '_' followed by a capital letter + PRAGMA_DISABLE_PUSH_CLANG( "-Wreserved-identifier" ) // identifier '%s' is reserved because it starts with '_' followed by a capital letter PRAGMA_DISABLE_PUSH_CLANG( "-Wcast-function-type" ) // cast from '%s' (aka '%s') to '%s' (aka '%s') converts to incompatible function type PRAGMA_DISABLE_PUSH_MSVC( 4191 ) // C4191: 'reinterpret_cast': unsafe conversion from 'FARPROC' to 'Type_CleanUp' PRAGMA_DISABLE_PUSH_MSVC( 4530 ) // C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc @@ -242,7 +242,7 @@ void VSProjectExternalNode::CopyConfigs() // GetProjectTypeGuid //------------------------------------------------------------------------------ -/*virtual*/ const AString & VSProjectExternalNode::GetProjectTypeGuid() const +/*virtual*/ const AString & VSProjectExternalNode::GetProjectTypeGuid() const { return m_ProjectTypeGuid; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp index 4d9dbf973..fc82ab480 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp @@ -90,12 +90,13 @@ REFLECT_END( XCodeProjectNode ) // CONSTRUCTOR //------------------------------------------------------------------------------ XCodeProjectNode::XCodeProjectNode() - : FileNode( AString::GetEmpty(), Node::FLAG_ALWAYS_BUILD ) + : FileNode() , m_XCodeOrganizationName( "Organization" ) , m_XCodeBuildToolPath( "./FBuild" ) , m_XCodeBuildToolArgs( "-ide $(FASTBUILD_TARGET)" ) , m_XCodeBuildWorkingDir( "./" ) { + m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_Type = Node::XCODEPROJECT_NODE; ProjectGeneratorBase::GetDefaultAllowedFileExtensions( m_ProjectAllowedFileExtensions ); diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp index 498c4f561..37650d6e8 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Args.cpp @@ -14,6 +14,8 @@ // system #if defined( __OSX__ ) #include +#elif defined( __LINUX__ ) + #include #endif // CONSTRUCTOR @@ -111,8 +113,10 @@ bool Args::Finalize( const AString & exe, const AString & nodeNameForError, Args #elif defined( __OSX__ ) const uint32_t argLimit( ARG_MAX - 1 ); #elif defined( __LINUX__ ) - // On Linux it's problematic to reliably determine this, so we make a best guess - const uint32_t argLimit( ( 128 * 1024 ) - 1 ); + // This doesn't account for some things that impact max arg length such + // as the size of the environment block, but it's better than using an + // arbitrary value + const uint32_t argLimit = static_cast( sysconf( _SC_ARG_MAX ) ); #endif // If the args exceed the cmd line limit, a response file is required diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.cpp index 73b1c541c..404bff449 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.cpp @@ -19,6 +19,7 @@ // External #include "lz4.h" #include "lz4hc.h" +#include "zstd.h" #include @@ -172,4 +173,108 @@ bool Compressor::Decompress( const void * data ) return false; } +// CompressZstd +//------------------------------------------------------------------------------ +bool Compressor::CompressZstd( const void * data, + size_t dataSize, + int32_t compressionLevel) +{ + PROFILE_FUNCTION; + + ASSERT( data ); + ASSERT( m_Result == nullptr ); + + // allocate worst case output size for LZ4 + const size_t worstCaseSize = ZSTD_compressBound( dataSize ); + UniquePtr output( (char *)ALLOC( worstCaseSize ) ); + + size_t compressedSize; + + // do compression + if ( compressionLevel > 0 ) + { + compressedSize = ZSTD_compress( output.Get(), + worstCaseSize, + static_cast( data ), + dataSize, + compressionLevel); + } + else + { + // Disable compression + compressedSize = dataSize; // Act as if compression achieved nothing + } + + // did the compression yield any benefit? + const bool compressed = ( compressedSize < dataSize ); + + if (compressed) + { + // trim memory usage to compressed size + m_Result = ALLOC( (uint32_t)compressedSize + sizeof(Header) ); + memcpy( (char *)m_Result + sizeof(Header), output.Get(), (size_t)compressedSize ); + m_ResultSize = (uint32_t)compressedSize + sizeof(Header); + } + else + { + // compression failed, so just copy the old data + m_Result = ALLOC( dataSize + sizeof(Header) ); + memcpy( (char *)m_Result + sizeof(Header), data, dataSize ); + m_ResultSize = dataSize + sizeof(Header); + } + + // fill out header + Header * header = (Header *)m_Result; + header->m_CompressionType = compressed ? 2u : 0u; // compression type + header->m_UncompressedSize = (uint32_t)dataSize; // input size + header->m_CompressedSize = compressed ? (uint32_t)compressedSize : (uint32_t)dataSize; // output size + + return compressed; +} + +// DecompressZstd +//------------------------------------------------------------------------------ +bool Compressor::DecompressZstd( const void * data ) +{ + PROFILE_FUNCTION; + + ASSERT( data ); + ASSERT( m_Result == nullptr ); + + const Header * header = static_cast(data); + + // handle uncompressed case + if (header->m_CompressionType == 0) + { + m_Result = ALLOC(header->m_UncompressedSize); + memcpy(m_Result, (const char *)data + sizeof(Header), header->m_UncompressedSize); + m_ResultSize = header->m_UncompressedSize; + return true; + } + ASSERT( header->m_CompressionType == 2 ); + + // uncompressed size + const uint32_t uncompressedSize = header->m_UncompressedSize; + m_Result = ALLOC( uncompressedSize ); + m_ResultSize = uncompressedSize; + + // skip over header to LZ4 data + const char * compressedData = ( (const char *)data + sizeof(Header) ); + + // decompress + const size_t bytesDecompressed = ZSTD_decompress( m_Result, + uncompressedSize, + compressedData, + header->m_CompressedSize ); + if ( bytesDecompressed == uncompressedSize ) + { + return true; + } + + // Data is corrupt + FREE( m_Result ); + m_Result = nullptr; + m_ResultSize = 0; + return false; +} //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.h b/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.h index 5c8bd68b4..7f4c8e4ed 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.h +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Compressor.h @@ -24,6 +24,10 @@ class Compressor bool Compress( const void * data, size_t dataSize, int32_t compressionLevel = -1 ); // -1 = default LZ4 compression level bool Decompress( const void * data ); + // Zstd + bool CompressZstd( const void * data, size_t dataSize, int32_t compressionLevel = -1 ); // -1 = default Zstd compression level + bool DecompressZstd( const void * data ); + const void * GetResult() const { return m_Result; } size_t GetResultSize() const { return m_ResultSize; } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ProjectGeneratorBase.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/ProjectGeneratorBase.cpp index e1616ad7a..9db7528aa 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ProjectGeneratorBase.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ProjectGeneratorBase.cpp @@ -613,7 +613,7 @@ void ProjectGeneratorBase::AddConfig( const ProjectGeneratorBaseConfig & config default: break; // Unsupported type - ignore } } - + return nullptr; } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Report/HTMLReport.h b/Code/Tools/FBuild/FBuildCore/Helpers/Report/HTMLReport.h index 8d2b4ad0b..7065f66a0 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Report/HTMLReport.h +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Report/HTMLReport.h @@ -52,7 +52,7 @@ class HTMLReport : public Report bool operator < ( const PieItem & other ) const { return m_Value > other.m_Value; } }; - + enum { DEFAULT_TABLE_WIDTH = 990 }; // Helpers diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp index c7c8fca0d..dacd9417c 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp @@ -30,7 +30,7 @@ JSONReport::~JSONReport() = default; void JSONReport::Generate( const NodeGraph & nodeGraph, const FBuildStats & stats ) { GetLibraryStats( nodeGraph, stats ); - + Write( "{\n\t" ); // build the report @@ -622,7 +622,7 @@ void JSONReport::DoIncludes() { Write( "]" ); } - else + else { Write( "\n\t]" ); } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Report/Report.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Report/Report.cpp index c0437e49a..53fe4093b 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Report/Report.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Report/Report.cpp @@ -307,7 +307,7 @@ Report::IncludeStatsMap::~IncludeStatsMap() Report::IncludeStats * Report::IncludeStatsMap::Find( const Node * node ) const { // caculate table entry - const uint32_t hash = node->GetNameCRC(); + const uint32_t hash = node->GetNameHash(); const uint32_t key = ( hash & 0xFFFF ); IncludeStats * item = m_Table[ key ]; @@ -330,7 +330,7 @@ Report::IncludeStats * Report::IncludeStatsMap::Find( const Node * node ) const Report::IncludeStats * Report::IncludeStatsMap::Insert( const Node * node ) { // caculate table entry - const uint32_t hash = node->GetNameCRC(); + const uint32_t hash = node->GetNameHash(); const uint32_t key = ( hash & 0xFFFF ); // insert new item diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp index f0ee99a2a..202c0c67d 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp @@ -296,7 +296,7 @@ bool ToolManifest::DeserializeFromRemote( IOStream & ms ) if ( !ms.Read( toolId ) || !ms.Read( mainExecutablePath ) || !ms.Read( numFiles ) || - ( AString::StrLen( mainExecutablePath.Get() ) != mainExecutablePath.GetLength() ) || + ( AString::StrLen( mainExecutablePath.Get() ) != mainExecutablePath.GetLength() ) || ( numFiles == 0 ) || // Must have at least 1 file ( toolId != m_ToolId ) ) // Must have correct toolId { @@ -607,7 +607,7 @@ bool ToolManifest::ReceiveFileData( uint32_t fileId, // Any replacement packet integrity validation should be not specific to // these packets and belongs at a higher level. static_assert( Protocol::PROTOCOL_VERSION_MAJOR == 22, "Remove backwards compat shims" ); - + // When running tests we should be using latest protocols which don't // have the bug anymore so this should never happen ASSERT( false && "Corrupt file data" ); // Catch errors during development @@ -701,7 +701,7 @@ bool ToolManifest::ReceiveFileData( uint32_t fileId, // Get path to file AStackString<> fileName; GetRemoteFilePath( fileId, fileName ); - + // Make modification time now FileIO::SetFileLastWriteTimeToNow( fileName ); } diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.h b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.h index efa78756e..05fc34041 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.h +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.h @@ -115,7 +115,7 @@ class ToolManifest : public Struct const char * GetRemoteEnvironmentString() const { return m_RemoteEnvironmentString; } static void GetRelativePath( const AString & root, const AString & otherFile, AString & otherFileRelativePath ); - + #if defined( __OSX__ ) || defined( __LINUX__ ) void TouchFiles() const; #endif diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp index 2fa4ccd16..7268077e0 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp @@ -73,10 +73,9 @@ void VSProjectGenerator::AddFile( const AString & file ) //------------------------------------------------------------------------------ void VSProjectGenerator::AddFiles( const Array< AString > & files ) { - const AString * const fEnd = files.End(); - for ( const AString * fIt = files.Begin(); fIt!=fEnd; ++fIt ) + for ( const AString & file : files ) { - AddFile( *fIt ); + AddFile( file ); } } @@ -115,12 +114,11 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile // Project Configurations { Write( " \n" ); - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF(" \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); - WriteF(" %s\n", cIt->m_Config.Get() ); - WriteF(" %s\n", cIt->m_Platform.Get() ); + WriteF(" \n", config.m_Config.Get(), config.m_Platform.Get() ); + WriteF(" %s\n", config.m_Config.Get() ); + WriteF(" %s\n", config.m_Platform.Get() ); Write( " \n" ); } Write( " \n" ); @@ -133,20 +131,19 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile { const AString & fileName = filePathPair.m_ProjectRelativePath; - const char * fileType = nullptr; - const VSProjectFileType * const end = fileTypes.End(); - for ( const VSProjectFileType * it=fileTypes.Begin(); it!=end; ++it ) + const char * fileTypeToUse = nullptr; + for ( const VSProjectFileType & fileType : fileTypes ) { - if ( AString::MatchI( it->m_Pattern.Get(), fileName.Get() ) ) + if ( AString::MatchI( fileType.m_Pattern.Get(), fileName.Get() ) ) { - fileType = it->m_FileType.Get(); + fileTypeToUse = fileType.m_FileType.Get(); break; } } - if ( fileType ) + if ( fileTypeToUse ) { WriteF( " \n", fileName.Get() ); - WriteF( " %s\n", fileType ); + WriteF( " %s\n", fileTypeToUse ); Write( " \n" ); } else @@ -162,10 +159,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile Write(" \n" ); { // Project References - const AString * const end = m_ProjectReferences.End(); - for ( const AString * it = m_ProjectReferences.Begin(); it != end; ++it ) + for ( const AString & projectReference : m_ProjectReferences ) { - AStackString<> proj( *it ); + AStackString<> proj( projectReference ); const char * pipe = proj.Find( '|' ); if ( pipe ) { @@ -183,10 +179,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile } { // References - const AString * const end = m_References.End(); - for ( const AString * it = m_References.Begin(); it != end; ++it ) + for ( const AString & reference : m_References ) { - WriteF( " \n", it->Get() ); + WriteF( " \n", reference.Get() ); } } Write(" \n" ); @@ -236,30 +231,29 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile // Configurations { - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF( " \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); + WriteF( " \n", config.m_Config.Get(), config.m_Platform.Get() ); Write( " Makefile\n" ); Write( " false\n" ); // If a specific executable is specified, use that, otherwise try to auto-derive // the executable from the .Target - AStackString<> localDebuggerCommand( cIt->m_LocalDebuggerCommand ); + AStackString<> localDebuggerCommand( config.m_LocalDebuggerCommand ); if ( localDebuggerCommand.IsEmpty() ) { // Get the executable path and make it project-relative - const Node * debugTarget = ProjectGeneratorBase::FindExecutableDebugTarget( cIt->m_TargetNode ); + const Node * debugTarget = ProjectGeneratorBase::FindExecutableDebugTarget( config.m_TargetNode ); if ( debugTarget ) { ProjectGeneratorBase::GetRelativePath( projectBasePath, debugTarget->GetName(), localDebuggerCommand ); } } - WritePGItem( "PlatformToolset", cIt->m_PlatformToolset ); - WritePGItem( "LocalDebuggerCommandArguments", cIt->m_LocalDebuggerCommandArguments ); + WritePGItem( "PlatformToolset", config.m_PlatformToolset ); + WritePGItem( "LocalDebuggerCommandArguments", config.m_LocalDebuggerCommandArguments ); WritePGItem( "LocalDebuggerCommand", localDebuggerCommand ); - WritePGItem( "LocalDebuggerEnvironment", cIt->m_LocalDebuggerEnvironment ); + WritePGItem( "LocalDebuggerEnvironment", config.m_LocalDebuggerEnvironment ); Write( " \n" ); } @@ -274,10 +268,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile // Property Sheets { - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF(" \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); + WriteF(" \n", config.m_Config.Get(), config.m_Platform.Get() ); Write( " \n" ); Write( " \n" ); } @@ -288,34 +281,37 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile // Property Group { - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF( " \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); + WriteF( " \n", config.m_Config.Get(), config.m_Platform.Get() ); - if ( cIt->m_Keyword == "Linux" ) + if ( config.m_Keyword == "Linux" ) { - WritePGItem( "BuildCommandLine", cIt->m_ProjectBuildCommand ); - WritePGItem( "ReBuildCommandLine", cIt->m_ProjectRebuildCommand ); - WritePGItem( "CleanCommandLine", cIt->m_ProjectCleanCommand ); + WritePGItem( "BuildCommandLine", config.m_ProjectBuildCommand ); + WritePGItem( "ReBuildCommandLine", config.m_ProjectRebuildCommand ); + WritePGItem( "CleanCommandLine", config.m_ProjectCleanCommand ); } else { - WritePGItem( "NMakeBuildCommandLine", cIt->m_ProjectBuildCommand ); - WritePGItem( "NMakeReBuildCommandLine", cIt->m_ProjectRebuildCommand ); - WritePGItem( "NMakeCleanCommandLine", cIt->m_ProjectCleanCommand ); + WritePGItem( "NMakeBuildCommandLine", config.m_ProjectBuildCommand ); + WritePGItem( "NMakeReBuildCommandLine", config.m_ProjectRebuildCommand ); + WritePGItem( "NMakeCleanCommandLine", config.m_ProjectCleanCommand ); + } + if ( !config.m_AndroidApkLocation.IsEmpty() ) + { + WritePGItem( "AndroidApkLocation", config.m_AndroidApkLocation ); } - WritePGItem( "NMakeOutput", cIt->m_Output ); + WritePGItem( "NMakeOutput", config.m_Output ); const ObjectListNode * oln = nullptr; - if ( cIt->m_PreprocessorDefinitions.IsEmpty() || cIt->m_IncludeSearchPath.IsEmpty() ) + if ( config.m_PreprocessorDefinitions.IsEmpty() || config.m_IncludeSearchPath.IsEmpty() ) { - oln = ProjectGeneratorBase::FindTargetForIntellisenseInfo( cIt->m_TargetNode ); + oln = ProjectGeneratorBase::FindTargetForIntellisenseInfo( config.m_TargetNode ); } - if ( cIt->m_PreprocessorDefinitions.IsEmpty() == false ) + if ( config.m_PreprocessorDefinitions.IsEmpty() == false ) { - WritePGItem( "NMakePreprocessorDefinitions", cIt->m_PreprocessorDefinitions ); + WritePGItem( "NMakePreprocessorDefinitions", config.m_PreprocessorDefinitions ); } else { @@ -334,9 +330,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile { ProjectGeneratorBase::ExtractIncludePaths( oln->GetCompilerOptions(), includePaths, forceIncludes, false ); } - if ( cIt->m_IncludeSearchPath.IsEmpty() == false ) + if ( config.m_IncludeSearchPath.IsEmpty() == false ) { - WritePGItem( "NMakeIncludeSearchPath", cIt->m_IncludeSearchPath ); + WritePGItem( "NMakeIncludeSearchPath", config.m_IncludeSearchPath ); } else if ( oln ) { @@ -351,9 +347,9 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile ProjectGeneratorBase::ConcatIntellisenseOptions( includePaths, includePathsStr, nullptr, ";" ); WritePGItem( "NMakeIncludeSearchPath", includePathsStr ); } - if ( cIt->m_ForcedIncludes.IsEmpty() == false ) + if ( config.m_ForcedIncludes.IsEmpty() == false ) { - WritePGItem( "NMakeForcedIncludes", cIt->m_ForcedIncludes ); + WritePGItem( "NMakeForcedIncludes", config.m_ForcedIncludes ); } else if ( oln ) { @@ -368,11 +364,11 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile ProjectGeneratorBase::ConcatIntellisenseOptions( forceIncludes, forceIncludePathsStr, nullptr, ";" ); WritePGItem( "NMakeForcedIncludes", forceIncludePathsStr ); } - WritePGItem( "NMakeAssemblySearchPath", cIt->m_AssemblySearchPath ); - WritePGItem( "NMakeForcedUsingAssemblies", cIt->m_ForcedUsingAssemblies ); - if ( cIt->m_AdditionalOptions.IsEmpty() == false ) + WritePGItem( "NMakeAssemblySearchPath", config.m_AssemblySearchPath ); + WritePGItem( "NMakeForcedUsingAssemblies", config.m_ForcedUsingAssemblies ); + if ( config.m_AdditionalOptions.IsEmpty() == false ) { - WritePGItem( "AdditionalOptions", cIt->m_AdditionalOptions ); + WritePGItem( "AdditionalOptions", config.m_AdditionalOptions ); } else { @@ -385,44 +381,43 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile WritePGItem( "AdditionalOptions", additionalOptionsStr ); } } - WritePGItem( "Xbox360DebuggerCommand", cIt->m_Xbox360DebuggerCommand ); - WritePGItem( "DebuggerFlavor", cIt->m_DebuggerFlavor ); - WritePGItem( "AumidOverride", cIt->m_AumidOverride ); - WritePGItem( "LocalDebuggerWorkingDirectory", cIt->m_LocalDebuggerWorkingDirectory ); - WritePGItem( "IntDir", cIt->m_IntermediateDirectory ); - WritePGItem( "OutDir", cIt->m_OutputDirectory ); - WritePGItem( "PackagePath", cIt->m_PackagePath ); - WritePGItem( "AdditionalSymbolSearchPaths", cIt->m_AdditionalSymbolSearchPaths ); - WritePGItem( "LayoutDir", cIt->m_LayoutDir ); - WritePGItem( "LayoutExtensionFilter", cIt->m_LayoutExtensionFilter ); - WritePGItem( "RemoteDebuggerCommand", cIt->m_RemoteDebuggerCommand ); - WritePGItem( "RemoteDebuggerCommandArguments", cIt->m_RemoteDebuggerCommandArguments ); - WritePGItem( "RemoteDebuggerWorkingDirectory", cIt->m_RemoteDebuggerWorkingDirectory ); + WritePGItem( "Xbox360DebuggerCommand", config.m_Xbox360DebuggerCommand ); + WritePGItem( "DebuggerFlavor", config.m_DebuggerFlavor ); + WritePGItem( "AumidOverride", config.m_AumidOverride ); + WritePGItem( "LocalDebuggerWorkingDirectory", config.m_LocalDebuggerWorkingDirectory ); + WritePGItem( "IntDir", config.m_IntermediateDirectory ); + WritePGItem( "OutDir", config.m_OutputDirectory ); + WritePGItem( "PackagePath", config.m_PackagePath ); + WritePGItem( "AdditionalSymbolSearchPaths", config.m_AdditionalSymbolSearchPaths ); + WritePGItem( "LayoutDir", config.m_LayoutDir ); + WritePGItem( "LayoutExtensionFilter", config.m_LayoutExtensionFilter ); + WritePGItem( "RemoteDebuggerCommand", config.m_RemoteDebuggerCommand ); + WritePGItem( "RemoteDebuggerCommandArguments", config.m_RemoteDebuggerCommandArguments ); + WritePGItem( "RemoteDebuggerWorkingDirectory", config.m_RemoteDebuggerWorkingDirectory ); Write( " \n" ); } } // ItemDefinition Groups { - const VSProjectConfig * const cEnd = configs.End(); - for ( const VSProjectConfig * cIt = configs.Begin(); cIt!=cEnd; ++cIt ) + for ( const VSProjectConfig & config : configs ) { - WriteF(" \n", cIt->m_Config.Get(), cIt->m_Platform.Get() ); + WriteF(" \n", config.m_Config.Get(), config.m_Platform.Get() ); Write( " \n" ); - if ( !cIt->m_BuildLogFile.IsEmpty() ) + if ( !config.m_BuildLogFile.IsEmpty() ) { - WritePGItem( "Path", cIt->m_BuildLogFile ); + WritePGItem( "Path", config.m_BuildLogFile ); } else { Write( " \n" ); } Write( " \n" ); - if ( ( !cIt->m_DeploymentType.IsEmpty() ) || ( !cIt->m_DeploymentFiles.IsEmpty() ) ) + if ( !config.m_DeploymentType.IsEmpty() || !config.m_DeploymentFiles.IsEmpty() ) { Write( " \n" ); - WritePGItem( "DeploymentType", cIt->m_DeploymentType ); - WritePGItem( "DeploymentFiles", cIt->m_DeploymentFiles ); + WritePGItem( "DeploymentType", config.m_DeploymentType ); + WritePGItem( "DeploymentFiles", config.m_DeploymentFiles ); Write( " \n" ); } Write( " \n" ); @@ -591,10 +586,8 @@ void VSProjectGenerator::WritePGItem( const char * xmlTag, const AString & value void VSProjectGenerator::GetFolderPath( const AString & fileName, AString & folder ) const { ASSERT( m_FilePathsCanonicalized ); - const AString * const bEnd = m_BasePaths.End(); - for ( const AString * bIt = m_BasePaths.Begin(); bIt != bEnd; ++bIt ) + for ( const AString & basePath : m_BasePaths ) { - const AString & basePath = *bIt; if ( fileName.BeginsWithI( basePath ) ) { const char * begin = fileName.Get() + basePath.GetLength(); diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/XCodeProjectGenerator.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/XCodeProjectGenerator.cpp index 118a85795..b6bd187d6 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/XCodeProjectGenerator.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/XCodeProjectGenerator.cpp @@ -83,7 +83,7 @@ const AString & XCodeProjectGenerator::GenerateUserSchemeMangementPList() "\t\t\tprimary\n" "\t\t\t\n" "\t\t\n" - "\t\n", + "\t\n", pbxNativeTargetGUID.Get() ); // Footer @@ -147,7 +147,7 @@ const AString & XCodeProjectGenerator::GenerateXCScheme() " \n" " \n" " \n", - pbxLegacyTargetGUID.Get(), m_ProjectName.Get(), m_ProjectName.Get(), m_ProjectName.Get() ); + pbxLegacyTargetGUID.Get(), m_ProjectName.Get(), m_ProjectName.Get(), m_ProjectName.Get() ); // Test Action m_Tmp.AppendFormat( " \n" " \n", - escapedArgument.Get() ); - } + escapedArgument.Get() ); + } m_Tmp.AppendFormat( " \n" ); } m_Tmp.AppendFormat( " \n" @@ -397,7 +397,7 @@ void XCodeProjectGenerator::WriteFolders() "\t\t\tchildren = (\n", pbxGroupGUID.Get() ); } - + // Child Files for ( const File * file : folder->m_Files ) { @@ -813,7 +813,7 @@ void XCodeProjectGenerator::WriteString( uint32_t indentDepth, { tabs += '\t'; } - + // Empty strings and strings with spaces are quoted const char quoteString = ShouldQuoteString( value ); const char * const formatString = quoteString ? "%s%s = \"%s\";\n" diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp index 8a5a3e971..85d9a2681 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp @@ -479,14 +479,14 @@ void Client::Process( const ConnectionInfo * connection, const Protocol::MsgRequ // If we will write the results to the cache, and this node is cacheable // then we want to respect higher cache compression levels if set const int16_t cacheCompressionLevel = FBuild::Get().GetOptions().m_CacheCompressionLevel; - if ( ( cacheCompressionLevel != 0 ) && - ( FBuild::Get().GetOptions().m_UseCacheWrite ) && + if ( ( cacheCompressionLevel != 0 ) && + ( FBuild::Get().GetOptions().m_UseCacheWrite ) && ( job->GetNode()->CastTo< ObjectNode >()->ShouldUseCache() ) ) { resultCompressionLevel = Math::Max( resultCompressionLevel, cacheCompressionLevel ); } } - + // Take note of the results compression level so we know to expect // compressed results job->SetResultCompressionLevel( resultCompressionLevel ); @@ -548,7 +548,7 @@ void Client::ProcessJobResultCommon( const ConnectionInfo * connection, bool isC uint32_t buildTime; ms.Read( buildTime ); - + uint16_t remoteThreadId = 0; ms.Read( remoteThreadId ); @@ -694,7 +694,7 @@ void Client::ProcessJobResultCommon( const ConnectionInfo * connection, bool isC if ( result == true ) { // built ok - serialize to disc - + ObjectNode * objectNode = node->CastTo< ObjectNode >(); // Store to cache if needed diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index cf5f0db68..fd2f81d49 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -163,13 +163,13 @@ bool Server::IsSynchingTool( AString & statusStr ) const // This is usually null here, but might need to be freed if // we had the connection drop between message and payload FREE( (void *)( cs->m_CurrentMessage ) ); - + // delete any jobs where we were waiting on Tool synchronization for ( Job * job : cs->m_WaitingJobs ) { delete job; } - + FDELETE cs; } } @@ -331,11 +331,11 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgJob // deserialize job ConstMemoryStream ms( payload, payloadSize ); - + Job * job = FNEW( Job( ms ) ); job->SetUserData( cs ); job->SetResultCompressionLevel( msg->GetResultCompressionLevel() ); - + // Get ToolId const uint64_t toolId = msg->GetToolId(); ASSERT( toolId ); @@ -370,8 +370,8 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgJob else { // Take ownership of toolchain - manifest->SetUserData( (void *)connection ); - + manifest->SetUserData( (void *)connection ); + const bool hasManifest = ( manifest->GetFiles().IsEmpty() == false ); if ( hasManifest ) { @@ -382,7 +382,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgJob { // Manifest was not sync'd. This can happen if disconnection // occurs before the manifest was received. - + // request manifest const Protocol::MsgRequestManifest reqMsg( toolId ); reqMsg.Send( connection ); @@ -435,7 +435,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgMani // when dealing with backwards compatibility with old workers) // If we ever break protocol compatibility, we can remove special handling static_assert( Protocol::PROTOCOL_VERSION_MAJOR == 22, "Remove backwards compat shims" ); - + // This should not happen with latest code so we want to catch that when // debugging ASSERT( false && "MsgManifest corrupt" ); @@ -496,7 +496,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgFile // when dealing with backwards compatibility with old workers) // If we ever break protocol compatibility, we can remove special handling static_assert( Protocol::PROTOCOL_VERSION_MAJOR == 22, "Remove backwards compat shims" ); - + // This should not happen with latest code so we want to catch that when // debugging ASSERT( false && "MsgFile corrupt" ); @@ -596,7 +596,7 @@ void Server::ThreadFunc() FinalizeCompletedJobs(); FindNeedyClients(); - + TouchToolchains(); JobQueueRemote::Get().MainThreadWait( 100 ); @@ -623,6 +623,15 @@ void Server::FindNeedyClients() ++availableJobs; // over request to parallelize building/network transfers + // determine job availability + int32_t availableJobs = (int32_t)WorkerThreadRemote::GetNumCPUsToUse(); + if ( availableJobs == 0 ) + { + return; + } + ++availableJobs; // over request to parallelize building/network transfers + + { MutexHolder mh( m_ClientListMutex ); @@ -727,9 +736,9 @@ void Server::FinalizeCompletedJobs() { ASSERT( cs->m_NumJobsActive.Load() > 0 ); cs->m_NumJobsActive.Decrement(); - + MutexHolder mh2( cs->m_Mutex ); - + if ( job->GetResultCompressionLevel() == 0 ) { // Uncompressed diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h index 8be313ed8..8b0643e0c 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.h @@ -94,7 +94,7 @@ class Server : public TCPConnectionPool mutable Mutex m_ToolManifestsMutex; Array< ToolManifest * > m_Tools; - + #if defined( __OSX__ ) || defined( __LINUX__ ) Timer m_TouchToolchainTimer; #endif diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp index e1f0ff13f..ea7d7dbf9 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.cpp @@ -20,14 +20,16 @@ // Static //------------------------------------------------------------------------------ static uint32_t s_LastJobId( 0 ); -/*static*/ int64_t Job::s_TotalLocalDataMemoryUsage( 0 ); +/*static*/ Atomic Job::s_TotalLocalDataMemoryUsage( 0 ); // CONSTRUCTOR //------------------------------------------------------------------------------ Job::Job( Node * node ) : m_Node( node ) { - m_JobId = AtomicInc( &s_LastJobId ); + // Constructor that assigns JobId can only be called on the main thread. + ASSERT( Thread::IsMainThread() ); + m_JobId = ++s_LastJobId; } // CONSTRUCTOR @@ -86,8 +88,7 @@ void Job::OwnData( void * data, size_t size, bool compressed ) // Update total memory use tracking if ( m_IsLocal ) { - ASSERT( s_TotalLocalDataMemoryUsage >= m_DataSize ); - AtomicSub( &s_TotalLocalDataMemoryUsage, (int64_t)m_DataSize ); + VERIFY( s_TotalLocalDataMemoryUsage.Sub( static_cast( m_DataSize ) ) >= 0 ); } } @@ -99,7 +100,7 @@ void Job::OwnData( void * data, size_t size, bool compressed ) // Update total memory use tracking if ( m_IsLocal ) { - AtomicAdd( &s_TotalLocalDataMemoryUsage, (int64_t)m_DataSize ); + s_TotalLocalDataMemoryUsage.Add( static_cast( m_DataSize ) ); } } @@ -266,7 +267,7 @@ void Job::GetMessagesForMonitorLog( AString & buffer ) const //------------------------------------------------------------------------------ /*static*/ uint64_t Job::GetTotalLocalDataMemoryUsage() { - return (uint64_t)AtomicLoadRelaxed( &s_TotalLocalDataMemoryUsage ); + return static_cast( s_TotalLocalDataMemoryUsage.Load() ); } // SetBuildProfilerScope diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h index e55b81ce9..42e3f7442 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/Job.h @@ -6,6 +6,7 @@ //------------------------------------------------------------------------------ #include "Core/Env/MSVCStaticAnalysis.h" #include "Core/Env/Types.h" +#include "Core/Process/Atomic.h" #include "Core/Strings/AString.h" // Forward Declarations @@ -124,7 +125,7 @@ class Job Array< AString > m_Messages; - static int64_t s_TotalLocalDataMemoryUsage; // Total memory being managed by OwnData + static Atomic s_TotalLocalDataMemoryUsage; // Total memory being managed by OwnData }; //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp index 1a868dd81..994ffa201 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.cpp @@ -16,7 +16,7 @@ #include "Core/Time/Timer.h" #include "Core/FileIO/FileIO.h" #include "Core/Process/Atomic.h" -#include "Core/Process/Thread.h" +#include "Core/Process/ThreadPool.h" #include "Core/Profile/Profile.h" // JobCostSorter @@ -34,7 +34,6 @@ class JobCostSorter //------------------------------------------------------------------------------ JobSubQueue::JobSubQueue() : m_Count( 0 ) - , m_Jobs( 1024, true ) { } @@ -57,6 +56,8 @@ uint32_t JobSubQueue::GetCount() const //------------------------------------------------------------------------------ void JobSubQueue::QueueJobs( Array< Node * > & nodes ) { + PROFILE_FUNCTION; + // Create wrapper Jobs around Nodes Array< Job * > jobs( nodes.GetSize() ); for ( Node * node : nodes ) @@ -73,16 +74,43 @@ void JobSubQueue::QueueJobs( Array< Node * > & nodes ) MutexHolder mh( m_Mutex ); const bool wasEmpty = m_Jobs.IsEmpty(); - m_Jobs.Append( jobs ); AtomicAdd( &m_Count, (uint32_t)jobs.GetSize() ); if ( wasEmpty ) { + m_Jobs.Swap( jobs ); return; // skip re-sorting } - // sort merged lists - m_Jobs.Sort( sorter ); + // Merge lists + Array< Job * > mergedList; + mergedList.SetSize( m_Jobs.GetSize() + jobs.GetSize() ); + Job ** dst = mergedList.Begin(); + Job ** src1 = m_Jobs.Begin(); + const Job * const * end1 = m_Jobs.End(); + Job ** src2 = jobs.Begin(); + const Job * const * end2 = jobs.End(); + while ( ( src1 < end1 ) && ( src2 < end2 ) ) + { + if ( sorter( *src1, *src2 ) ) + { + *dst++ = *src1++; + } + else + { + *dst++ = *src2++; + } + } + while ( src1 < end1 ) + { + *dst++ = *src1++; + } + while ( src2 < end2 ) + { + *dst++ = *src2++; + } + ASSERT( dst == mergedList.End() ); + m_Jobs.Swap( mergedList ); } // RemoveJob @@ -114,7 +142,7 @@ Job * JobSubQueue::RemoveJob() // CONSTRUCTOR //------------------------------------------------------------------------------ -JobQueue::JobQueue( uint32_t numWorkerThreads ) : +JobQueue::JobQueue( uint32_t numWorkerThreads, ThreadPool * threadPool ) : m_NumLocalJobsActive( 0 ), m_DistributableJobs_Available( 1024, true ), m_DistributableJobs_InProgress( 1024, true ), @@ -133,14 +161,18 @@ JobQueue::JobQueue( uint32_t numWorkerThreads ) : WorkerThread::InitTmpDir(); - for ( uint32_t i=0; i 0 ) { - // identify each worker with an id starting from 1 - // (the "main" thread is considered 0) - const uint16_t threadIndex = static_cast( i + 1 ); - WorkerThread * wt = FNEW( WorkerThread( threadIndex ) ); - wt->Init(); - m_Workers.Append( wt ); + // Create a job to run on each thread + for ( uint32_t i = 0; i < numWorkerThreads; ++i ) + { + // identify each worker with an id starting from 1 + // (the "main" thread is considered 0) + const uint16_t threadIndex = static_cast( i + 1 ); + WorkerThread * wt = FNEW( WorkerThread( threadIndex ) ); + wt->Init( threadPool ); + m_Workers.Append( wt ); + } } } @@ -251,7 +283,7 @@ bool JobQueue::HasPendingCompletedJobs() const //------------------------------------------------------------------------------ void JobQueue::AddJobToBatch( Node * node ) { - ASSERT( node->GetState() == Node::DYNAMIC_DEPS_DONE ); + ASSERT( node->GetState() == Node::DYNAMIC_DEPS ); // mark as building node->SetState( Node::BUILDING ); @@ -797,7 +829,7 @@ void JobQueue::FinishedProcessingJob( Job * job, bool success, bool wasARemoteJo { // nothing to check } - else + else { // build completed ok, or retrieved from cache... ASSERT( ( result == Node::NODE_RESULT_OK ) || ( result == Node::NODE_RESULT_OK_CACHE ) ); diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h index d8e96c8d5..4a3e4f80d 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueue.h @@ -15,6 +15,7 @@ //------------------------------------------------------------------------------ class Node; class Job; +class ThreadPool; class WorkerThread; @@ -44,7 +45,7 @@ class JobSubQueue class JobQueue : public Singleton< JobQueue > { public: - explicit JobQueue( uint32_t numWorkerThreads ); + explicit JobQueue( uint32_t numWorkerThreads, ThreadPool * threadPool ); ~JobQueue(); // main thread calls these diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp index 2774a4e90..bbba846c8 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp @@ -19,6 +19,7 @@ #include "Core/FileIO/FileIO.h" #include "Core/FileIO/FileStream.h" #include "Core/FileIO/PathUtils.h" +#include "Core/Process/ThreadPool.h" #include "Core/Profile/Profile.h" #include "Core/Time/Timer.h" #include "Core/Tracing/Tracing.h" @@ -33,13 +34,17 @@ JobQueueRemote::JobQueueRemote( uint32_t numWorkerThreads ) : { WorkerThread::InitTmpDir( true ); // remote == true + // Create thread pool + m_ThreadPool = FNEW( ThreadPool( numWorkerThreads ) ); + + // Create a job to run on each thread for ( uint32_t i=0; i( i + 1001 ); WorkerThread * wt = FNEW( WorkerThreadRemote( threadIndex ) ); - wt->Init(); + wt->Init( m_ThreadPool ); m_Workers.Append( wt ); } } @@ -58,6 +63,8 @@ JobQueueRemote::~JobQueueRemote() m_Workers[ i ]->WaitForStop(); FDELETE m_Workers[ i ]; } + + FDELETE m_ThreadPool; } // SignalStopWorkers (Main Thread) diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.h index 4ec44799f..955b8dd80 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.h @@ -15,6 +15,7 @@ //------------------------------------------------------------------------------ class Node; class Job; +class ThreadPool; class WorkerThread; // JobQueueRemote @@ -67,6 +68,7 @@ class JobQueueRemote : public Singleton< JobQueueRemote > Semaphore m_WorkerThreadSemaphore; Semaphore m_WorkerThreadSleepSemaphore; + ThreadPool * m_ThreadPool; Array< WorkerThread * > m_Workers; }; diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerBrokerageServer.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerBrokerageServer.cpp index 2eb582244..c5e74f31b 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerBrokerageServer.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerBrokerageServer.cpp @@ -66,7 +66,7 @@ void WorkerBrokerageServer::SetAvailability( bool available ) const float elapsedTime = m_TimerLastUpdate.GetElapsed(); if ( elapsedTime >= sBrokerageAvailabilityUpdateTime ) { - // If settings have changed, (re)create the file + // If settings have changed, (re)create the file // If settings have not changed, update the modification timestamp const WorkerSettings & workerSettings = WorkerSettings::Get(); const uint64_t settingsWriteTime = workerSettings.GetSettingsWriteTime(); diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp index 95bea6cf3..daf6b604b 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp @@ -17,7 +17,7 @@ #include "Core/FileIO/FileStream.h" #include "Core/FileIO/PathUtils.h" #include "Core/Process/Atomic.h" -#include "Core/Process/Thread.h" +#include "Core/Process/ThreadPool.h" #include "Core/Profile/Profile.h" // Static @@ -36,12 +36,11 @@ WorkerThread::WorkerThread( uint16_t threadIndex ) // Init //------------------------------------------------------------------------------ -void WorkerThread::Init() +void WorkerThread::Init( ThreadPool * pool ) { PROFILE_FUNCTION; - // Start thread - m_Thread.Start( ThreadWrapperFunc, "WorkerThread", this, MEGABYTE ); + pool->EnqueueJob( ThreadWrapperFunc, this ); } //------------------------------------------------------------------------------ @@ -107,19 +106,12 @@ void WorkerThread::WaitForStop() // MainWrapper //------------------------------------------------------------------------------ -/*static*/ uint32_t WorkerThread::ThreadWrapperFunc( void * param ) +/*static*/ void WorkerThread::ThreadWrapperFunc( void * param ) { WorkerThread * wt = static_cast< WorkerThread * >( param ); s_WorkerThreadThreadIndex = wt->m_ThreadIndex; - #if defined( PROFILING_ENABLED ) - AStackString<> threadName; - threadName.Format( "%s_%02u", s_WorkerThreadThreadIndex > 1000 ? "RemoteWorkerThread" : "WorkerThread", s_WorkerThreadThreadIndex ); - PROFILE_SET_THREAD_NAME( threadName.Get() ); - #endif - wt->Main(); - return 0; } // Main diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h index 61ef5b3ad..3149bed3f 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h @@ -15,6 +15,7 @@ // Forward Declarations //------------------------------------------------------------------------------ class FileStream; +class ThreadPool; // WorkerThread //------------------------------------------------------------------------------ @@ -22,7 +23,7 @@ class WorkerThread { public: explicit WorkerThread( uint16_t threadIndex ); - void Init(); + void Init( ThreadPool * pool ); virtual ~WorkerThread(); static void InitTmpDir( bool remote = false ); @@ -46,7 +47,7 @@ class WorkerThread static bool Update(); // worker thread main loop - static uint32_t ThreadWrapperFunc( void * param ); + static void ThreadWrapperFunc( void * param ); virtual void Main(); // signal to exit thread diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/SubDir/subdir.bff b/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/SubDir/subdir.bff index c604bc41a..4a34252ec 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/SubDir/subdir.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/SubDir/subdir.bff @@ -11,4 +11,4 @@ Print( 'OK-2B' ) // Should not see file in root (one level up) #if !file_exists( "if_file_exists_directive.bff" ) Print( 'OK-4' ) -#endif \ No newline at end of file +#endif diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/if_file_exists_directive.bff b/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/if_file_exists_directive.bff index 49c8e0aeb..6c44fe449 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/if_file_exists_directive.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestBFFParsing/if_file_exists_directive.bff @@ -6,4 +6,4 @@ Print("File exists") #else Print("File does not exist") -#endif \ No newline at end of file +#endif diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/common.h b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/common.h index 4314c228d..3e2f55090 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/common.h +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/common.h @@ -1 +1 @@ -#include "file.h" \ No newline at end of file +#include "file.h" diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/fbuild.bff index 4734f4222..029cc586a 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeHierarchy/fbuild.bff @@ -11,7 +11,7 @@ Settings {} // use Standard Environment ObjectList( 'ObjectList' ) { - .CompilerInputFiles = { + .CompilerInputFiles = { '$TestRoot$/Data/TestCache/LightCache_IncludeHierarchy/Folder1/file.cpp' '$TestRoot$/Data/TestCache/LightCache_IncludeHierarchy/Folder2/file.cpp' } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro/fbuild.bff index f05637663..9b560117b 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro/fbuild.bff @@ -10,7 +10,7 @@ Settings {} // use Standard Environment ObjectList( 'ObjectList' ) { - .CompilerInputFiles = { + .CompilerInputFiles = { '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro/file.1.cpp' '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro/file.2.cpp' } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/fbuild.bff index 292dab4bb..66d88d859 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/fbuild.bff @@ -10,7 +10,7 @@ Settings {} // use Standard Environment ObjectList( 'ObjectList' ) { - .CompilerInputFiles = { + .CompilerInputFiles = { '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp' '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp' } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp index e9f650537..1837eb084 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.1.cpp @@ -3,4 +3,4 @@ #include "header1.h" // now we include via the macro -#include INCLUDE_VIA_MACRO +#include INCLUDE_VIA_MACRO diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp index e9f650537..1837eb084 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro2/file.2.cpp @@ -3,4 +3,4 @@ #include "header1.h" // now we include via the macro -#include INCLUDE_VIA_MACRO +#include INCLUDE_VIA_MACRO diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro3/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro3/fbuild.bff index 6dbb75d8b..0b6adca9d 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro3/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestCache/LightCache_IncludeUsingMacro3/fbuild.bff @@ -10,7 +10,7 @@ Settings {} // use Standard Environment ObjectList( 'ObjectList' ) { - .CompilerInputFiles = { + .CompilerInputFiles = { '$TestRoot$/Data/TestCache/LightCache_IncludeUsingMacro3/file.cpp' } .CompilerOutputPath = '$Out$/Test/Cache/LightCache_IncludeUsingMacro3/' diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestDistributed/RemoteRaceSystemFailure/Fast.cpp b/Code/Tools/FBuild/FBuildTest/Data/TestDistributed/RemoteRaceSystemFailure/Fast.cpp index 07a18511a..402388caa 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestDistributed/RemoteRaceSystemFailure/Fast.cpp +++ b/Code/Tools/FBuild/FBuildTest/Data/TestDistributed/RemoteRaceSystemFailure/Fast.cpp @@ -1 +1 @@ -// No code needed - test functions with an empty object file \ No newline at end of file +// No code needed - test functions with an empty object file diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff new file mode 100644 index 000000000..0f7b91298 --- /dev/null +++ b/Code/Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff @@ -0,0 +1,16 @@ +// +// DatabaseLocation +// +//------------------------------------------------------------------------------ + +// Use the standard test environment +//------------------------------------------------------------------------------ +#include "../../testcommon.bff" +Using( .StandardEnvironment ) +Settings {} + +TextFile( 'TestTarget' ) +{ + .TextFileOutput = '$Out$/Test/Graph/DatabaseLocation/test.txt' + .TextFileInputStrings = { 'Test content' } +} diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/Exclusions/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/Exclusions/fbuild.bff index 4358796d3..cdc8803ac 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/Exclusions/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestObjectList/Exclusions/fbuild.bff @@ -87,4 +87,4 @@ Alias( 'Test' ) 'ExcludePattern-ForwardSlash' 'ExcludePattern-Backslash' } -} \ No newline at end of file +} diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff index 9224da58d..e71150069 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff @@ -64,10 +64,10 @@ Executable( "PCHTest" ) ObjectList( "PCHTestClang-Windows" ) { Using( .ToolChain_Clang_Windows ) - + .PCHInputFile = "Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/PrecompiledHeader.cpp" .PCHOutputFile = "$Out$\Test\PrecompiledHeaders\Clang-Windows\PrecompiledHeader.pch" - + .CompilerOptions + ' /Yu"PrecompiledHeader.h" /Fp"$PCHOutputFile$"' + ' "/ITools/FBuild/FBuildTest/Data/TestPrecompiledHeaders"' + ' -Wno-unused-macros' diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild.bff index bf116db50..e7480aedd 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild.bff @@ -54,7 +54,7 @@ VSSolution( 'ExternalProjectSolution' ) // only uncomment this if you have VSProjTypeExtractor / Visual Studio with C# support available (which is not likely the case on CI) ;.SolutionProjects + { 'External', 'ExternalMissingBraces' } - + // only uncomment this if you have VSProjTypeExtractor / Visual Studio with Python support available (which is not likely the case on CI) ;.SolutionProjects + { 'ExternalDummyProject_2' } diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild_WrongData.bff b/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild_WrongData.bff index 4d85fd3df..20e34f4a0 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild_WrongData.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestProjectGeneration/Solution_ExternalProject/fbuild_WrongData.bff @@ -24,10 +24,10 @@ VSSolution( 'ExternalWrongDataProjectSolution' ) { .SolutionProjects = { 'Proj1w', 'Proj2w' } - // this will always fail, even if you have VSProjTypeExtractor / Visual Studio with WiX toolset support available (which is not likely the case on CI) - // and should do so, that's what this test validates - .SolutionProjects + { 'ExternalDummyProject_3' } - + // this will always fail, even if you have VSProjTypeExtractor / Visual Studio with WiX toolset support available (which is not likely the case on CI) + // and should do so, that's what this test validates + .SolutionProjects + { 'ExternalDummyProject_3' } + .SolutionX64Debug = [ .Platform = 'x64' .Config = 'Debug' ] .SolutionX64Release = [ .Platform = 'x64' .Config = 'Release' .SolutionDeployProjects = 'Proj1w' ] .SolutionConfigs = { .SolutionX64Debug, .SolutionX64Release } diff --git a/Code/Tools/FBuild/FBuildTest/FBuildTest.bff b/Code/Tools/FBuild/FBuildTest/FBuildTest.bff index 509e41c1a..d5f694015 100644 --- a/Code/Tools/FBuild/FBuildTest/FBuildTest.bff +++ b/Code/Tools/FBuild/FBuildTest/FBuildTest.bff @@ -55,6 +55,7 @@ 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' 'xxHash-Lib-$Platform$-$BuildConfigName$' + 'Zstd-Lib-$Platform$-$BuildConfigName$' } .LinkerOutput = '$OutputBase$/$ProjectPath$/$ProjectName$$ExeExtension$' #if __WINDOWS__ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h index f5acea659..fcfbcb296 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h +++ b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.h @@ -94,6 +94,8 @@ class FBuildForTest : public FBuild void SerializeDepGraphToText( const char * nodeName, AString & outBuffer ) const; + const AString & GetDependencyGraphFile() const { return m_DependencyGraphFile; } + using FBuild::Build; virtual bool Build( Node * nodeToBuild ) override; }; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp index c17550bcc..ef69b4b21 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp @@ -144,8 +144,8 @@ void TestArgs::Check( ArgsResponseFileMode mode, if ( longArgs ) { - // Add ~3200 KiB of command line args ( 32 * 100 * 1024 ) - for ( size_t i = 0; i < 100 * 1024; ++i ) + // Add ~32 MiB of command line args ( 32 * 1024 * 1024 ) + for ( size_t i = 0; i < 1024 * 1024; ++i ) { args += "123456789012345678901234567890X"; // 31 chars args.AddDelimiter(); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp index 6794d743f..16004de70 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp @@ -81,6 +81,7 @@ class TestBFFParsing : public FBuildTest void ErrorRowAndColumn() const; void ForEach() const; void FunctionHeaders() const; + void AlreadyDefined() const; }; // Register Tests @@ -115,7 +116,7 @@ REGISTER_TESTS_BEGIN( TestBFFParsing ) REGISTER_TEST( IfExistsDirective ) REGISTER_TEST( IfFileExistsDirective ) REGISTER_TEST( IfFileExistsDirective_RelativePaths ) - REGISTER_TEST( IfBooleanOperators ) + REGISTER_TEST( IfBooleanOperators ) REGISTER_TEST( ElseDirective ) REGISTER_TEST( ElseDirective_Bad ) REGISTER_TEST( ElseDirective_Bad2 ) @@ -145,6 +146,7 @@ REGISTER_TESTS_BEGIN( TestBFFParsing ) REGISTER_TEST( ErrorRowAndColumn ) REGISTER_TEST( ForEach ) REGISTER_TEST( FunctionHeaders ) + REGISTER_TEST( AlreadyDefined ) REGISTER_TESTS_END // Empty @@ -543,7 +545,7 @@ void TestBFFParsing::IfFileExistsDirective_RelativePaths() const { // file_exists treats paths the same way as #include // (paths are relative to the bff) - + FBuildTestOptions options; options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestBFFParsing/IfFileExistsDirective/RelativePaths/root.bff"; @@ -1097,3 +1099,32 @@ void TestBFFParsing::FunctionHeaders() const } //------------------------------------------------------------------------------ +void TestBFFParsing::AlreadyDefined() const +{ + // Alias + TEST_PARSE_FAIL( "Alias( 'X' ) { .Targets = 'file' }\n" + "Alias( 'X' ) { .Targets = 'file' }\n", + "Error #1100 - Previously declared here:" ); + + // Alias for a Function + TEST_PARSE_FAIL( "Test( 'X' ) { .TestExecutable = 'exe' .TestOutput = 'out1' }\n" + "Test( 'X' ) { .TestExecutable = 'exe' .TestOutput = 'out2' }\n", + "Error #1100 - Previously declared here:" ); + + // Direct name (not an Alias) + TEST_PARSE_FAIL( "ObjectList( 'X' ) { .Compiler = 'clang' .CompilerOptions = '%1 %2' }\n" + "ObjectList( 'X' ) { .Compiler = 'clang' .CompilerOptions = '%1 %2' }\n", + "Error #1100 - Previously declared here:" ); + + // Aliasing output + TEST_PARSE_FAIL( "Alias( 'X' ) { .Targets = 'file' }\n" + "TextFile() { .TextFileOutput = 'file' }\n", + "Error #1100 - Previously declared here:" ); + + // Same target created multiple times in a loop + TEST_PARSE_FAIL( ".Y = { 'a', 'b' }\n" + "ForEach( .X in .Y ) { Test() { .TestExecutable = 'exe' .TestOutput = 'out' } }\n", + "Error #1100 - Previously declared here:" ); +} + +//------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp index fb625ff9e..116f82b37 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp @@ -908,9 +908,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_Write() const TEST_ASSERT( output.Find( "warning C6201" ) && output.Find( "Index '32' is out of valid index range" ) ); TEST_ASSERT( output.Find( "warning C6386" ) && output.Find( "Buffer overrun while writing to 'buffer'" ) ); // file2.cpp - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( output.Find( "warning C6387" ) && output.Find( "could be '0'" ) ); - #endif + TEST_ASSERT( output.Find( "warning C6387" ) && output.Find( "could be '0'" ) ); // Check analysis file is present with expected errors AString xml; @@ -918,9 +916,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_Write() const TEST_ASSERT( xml.Find( "6201" ) ); TEST_ASSERT( xml.Find( "6386" ) ); LoadFileContentsAsString( mAnalyzeMSVCXMLFile2, xml ); - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( xml.Find( "6387" ) ); - #endif + TEST_ASSERT( xml.Find( "6387" ) ); } // Analyze_MSVC_WarningsOnly_Read @@ -952,9 +948,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_Read() const TEST_ASSERT( xml.Find( "6201" ) ); TEST_ASSERT( xml.Find( "6386" ) ); LoadFileContentsAsString( mAnalyzeMSVCXMLFile2, xml ); - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( xml.Find( "6387" ) ); - #endif + TEST_ASSERT( xml.Find( "6387" ) ); } // Analyze_MSVC_WarningsOnly_WriteFromDist @@ -993,9 +987,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_WriteFromDist() const TEST_ASSERT( output.Find( "warning C6201" ) && output.Find( "Index '32' is out of valid index range" ) ); TEST_ASSERT( output.Find( "warning C6386" ) && output.Find( "Buffer overrun while writing to 'buffer'" ) ); // file2.cpp - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( output.Find( "warning C6387" ) && output.Find( "could be '0': this does not adhere to the specification for the function" ) ); - #endif + TEST_ASSERT( output.Find( "warning C6387" ) && output.Find( "could be '0': this does not adhere to the specification for the function" ) ); // Check analysis file is present with expected errors AString xml; @@ -1003,9 +995,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_WriteFromDist() const TEST_ASSERT( xml.Find( "6201" ) ); TEST_ASSERT( xml.Find( "6386" ) ); LoadFileContentsAsString( mAnalyzeMSVCXMLFile2, xml ); - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( xml.Find( "6387" ) ); - #endif + TEST_ASSERT( xml.Find( "6387" ) ); } // Analyze_MSVC_WarningsOnly_ReadFromDist @@ -1046,9 +1036,7 @@ void TestCache::Analyze_MSVC_WarningsOnly_ReadFromDist() const TEST_ASSERT( xml.Find( "6201" ) ); TEST_ASSERT( xml.Find( "6386" ) ); LoadFileContentsAsString( mAnalyzeMSVCXMLFile2, xml ); - #if defined( _MSC_VER ) && ( _MSC_VER >= 1910 ) // From VS2017 or later - TEST_ASSERT( xml.Find( "6387" ) ); - #endif + TEST_ASSERT( xml.Find( "6387" ) ); } // ExtraFiles diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp index 41e1e5bf6..77cb23277 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp @@ -31,7 +31,8 @@ class TestCompressor : public FBuildTest void CompressSimpleHelper( const char * data, size_t size, size_t expectedCompressedSize, - bool shouldCompress ) const; + bool shouldCompress, + bool useZstd = false ) const; void CompressHelper( const char * fileName ) const; }; @@ -76,7 +77,8 @@ void TestCompressor::CompressSimple() const void TestCompressor::CompressSimpleHelper( const char * data, size_t size, size_t expectedCompressedSize, - bool shouldCompress ) const + bool shouldCompress, + bool useZstd ) const { // raw input strings may not be aligned on Linux/OSX, so copy // them to achieve our required alignment @@ -86,7 +88,8 @@ void TestCompressor::CompressSimpleHelper( const char * data, // compress Compressor c; - const bool compressed = c.Compress( data, size ); + const bool compressed = useZstd ? c.CompressZstd( data, size ) + : c.Compress( data, size ); TEST_ASSERT( compressed == shouldCompress ); const size_t compressedSize = c.GetResultSize(); if ( expectedCompressedSize > 0 ) @@ -97,7 +100,14 @@ void TestCompressor::CompressSimpleHelper( const char * data, // decompress Compressor d; - TEST_ASSERT( d.Decompress( compressedMem ) ); + if ( useZstd ) + { + TEST_ASSERT( d.DecompressZstd( compressedMem ) ); + } + else + { + TEST_ASSERT( d.Decompress( compressedMem ) ); + } const size_t decompressedSize = d.GetResultSize(); TEST_ASSERT( decompressedSize == size ); TEST_ASSERT( memcmp( data, d.GetResult(), size ) == 0 ); @@ -140,9 +150,11 @@ void TestCompressor::CompressHelper( const char * fileName ) const OUTPUT( "Level | Time (ms) MB/s Ratio | Time (ms) MB/s\n" ); OUTPUT( "------------------------------------------------\n" ); + OUTPUT( "LZ4:\n" ); + // Compress at various compression levels const int32_t compressionLevels[] = - { + { 0, // Disabled -256, -128, -64, -32, -16, -8, -4, -2, -1, // LZ4 1, 3, 6, 9, 12 // LZ4 HC @@ -193,6 +205,61 @@ void TestCompressor::CompressHelper( const char * fileName ) const ( compressTimeTaken / numRepeats ), compressThroughputMBs, (double)ratio, ( decompressTimeTaken / numRepeats ), decompressThroughputMBs ); } + + OUTPUT( "Zstd:\n" ); + + // Compress at various compression levels + const int32_t zStdCompressionLevels[] = + { + 0, // Disabled + 1, 3, 6, 9, 12, 15, 18, 21 // Zstd + }; + + for ( const int32_t compressionLevel : zStdCompressionLevels ) + { + // compress/decompress the data several times to get more stable throughput value + const uint32_t numRepeats = 4; // Increase to get more consistent numbers + double compressTimeTaken = 0.0; + double decompressTimeTaken = 0.0; + uint64_t compressedSize = 0; + + // Compression speed + UniquePtr< Compressor, DeleteDeletor > c; + for ( uint32_t i = 0; i < numRepeats; ++i ) + { + // Compress + c = FNEW( Compressor ); + const Timer t; + c.Get()->CompressZstd( data.Get(), dataSize, compressionLevel ); + compressedSize = c.Get()->GetResultSize(); + compressTimeTaken += (double)t.GetElapsedMS(); + } + + // Decompression speed + for ( uint32_t i = 0; i < numRepeats; ++i ) + { + // Decompress + const Timer t2; + Compressor d; + TEST_ASSERT( d.DecompressZstd( c.Get()->GetResult() ) ); + TEST_ASSERT( d.GetResultSize() == dataSize ); + decompressTimeTaken += (double)t2.GetElapsedMS(); + + // Sanity check decompression returns original results + if ( i == 0 ) + { + TEST_ASSERT( memcmp( data.Get(), d.GetResult(), dataSize ) == 0 ); + } + } + + const double compressThroughputMBs = ( ( (double)dataSize * (double)numRepeats ) / ( compressTimeTaken / 1000.0 ) ) / (double)MEGABYTE; + const double decompressThroughputMBs = ( ( (double)dataSize * (double)numRepeats ) / ( decompressTimeTaken / 1000.0 ) ) / (double)MEGABYTE; + const double ratio = ( (double)dataSize / (double)compressedSize ); + + OUTPUT( "%-5i | %8.3f %7.1f %5.2f | %8.3f %7.1f\n", compressionLevel, + ( compressTimeTaken / numRepeats ), compressThroughputMBs, (double)ratio, + ( decompressTimeTaken / numRepeats ), decompressThroughputMBs ); + } OUTPUT( "------------------------------------------------\n" ); } diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestDependencies.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestDependencies.cpp index c4ec59e9f..af4f7201d 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestDependencies.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestDependencies.cpp @@ -64,7 +64,7 @@ void TestDependencies::Add() const // Node with defaults { Dependencies d; - d.Add( nodes[ 0 ] ); + d.Add( nodes[ 0 ] ); TEST_ASSERT( d.IsEmpty() == false ); TEST_ASSERT( d.GetSize() == 1 ); TEST_ASSERT( d.GetCapacity() > 0 ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp index 3357a908c..837312b1e 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestExe.cpp @@ -19,7 +19,6 @@ class TestExe : public FBuildTest private: DECLARE_TESTS - void CreateNode() const; void Build() const; void CheckValidExe() const; void Build_NoRebuild() const; @@ -28,30 +27,11 @@ class TestExe : public FBuildTest // Register Tests //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestExe ) - REGISTER_TEST( CreateNode ) REGISTER_TEST( Build ) REGISTER_TEST( CheckValidExe ) REGISTER_TEST( Build_NoRebuild ) REGISTER_TESTS_END -// CreateNode -//------------------------------------------------------------------------------ -void TestExe::CreateNode() const -{ - FBuild fb; - NodeGraph ng; - - #if defined( __WINDOWS__ ) - AStackString<> exeName( "c:\\exe.exe" ); - #else - AStackString<> exeName( "/tmp/exe.exe" ); - #endif - const ExeNode * exeNode = ng.CreateExeNode( exeName ); - TEST_ASSERT( exeNode->GetType() == Node::EXE_NODE ); - TEST_ASSERT( ExeNode::GetTypeS() == Node::EXE_NODE ); - TEST_ASSERT( AStackString<>( "Exe" ) == exeNode->GetTypeName() ); -} - // Build //------------------------------------------------------------------------------ void TestExe::Build() const diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp index 2d3ed395f..71ef3a07a 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp @@ -322,15 +322,14 @@ void TestExec::Build_ExecEnvCommand() const FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); - // build (via alias) - TEST_ASSERT( fBuild.Build( "EnvHelperExe" ) ); + // Build and run exe that checks env var is set + // (the executable checks the expected env var) + TEST_ASSERT( fBuild.Build( "ExecEnvCommandTest" ) ); // Check stats // Seen, Built, Type CheckStatsNode ( 1, 1, Node::EXE_NODE ); - - // Run the execenv command and ensure we get the expected output - TEST_ASSERT( fBuild.Build( "ExecEnvCommandTest" ) ); + CheckStatsNode ( 1, 1, Node::EXEC_NODE ); } // Exclusions diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp index 8fba02ce2..f77408d20 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp @@ -19,7 +19,9 @@ #include "Tools/FBuild/FBuildCore/Graph/LibraryNode.h" #include "Tools/FBuild/FBuildCore/Graph/NodeGraph.h" #include "Tools/FBuild/FBuildCore/Graph/ObjectNode.h" +#include "Tools/FBuild/FBuildCore/Graph/RemoveDirNode.h" #include "Tools/FBuild/FBuildCore/Graph/SettingsNode.h" +#include "Tools/FBuild/FBuildCore/Graph/TestNode.h" #include "Tools/FBuild/FBuildCore/Graph/UnityNode.h" // Core @@ -57,6 +59,7 @@ class TestGraph : public FBuildTest void DBVersionChanged() const; void FixupErrorPaths() const; void CyclicDependency() const; + void DBLocation() const; }; // Register Tests @@ -78,6 +81,7 @@ REGISTER_TESTS_BEGIN( TestGraph ) REGISTER_TEST( DBVersionChanged ) REGISTER_TEST( FixupErrorPaths ) REGISTER_TEST( CyclicDependency ) + REGISTER_TEST( DBLocation ) REGISTER_TESTS_END // NodeTestHelper @@ -88,8 +92,10 @@ class NodeTestHelper : public Node REFLECT_DECLARE( NodeTestHelper ) public: NodeTestHelper() - : Node( AStackString<>( "dummy" ), Node::PROXY_NODE, 0 ) - {} + : Node( Node::PROXY_NODE ) + { + SetName( AStackString<>( "placeholder" ) ); + } virtual bool Initialize( NodeGraph & /*nodeGraph*/, const BFFToken * /*funcStartIter*/, const Function * /*function*/ ) override { ASSERT( false ); @@ -116,114 +122,41 @@ void TestGraph::TestNodeTypes() const FBuild fb; NodeGraph ng; - const FileNode * fn = ng.CreateFileNode( AStackString<>( "file" ) ); - TEST_ASSERT( fn->GetType() == Node::FILE_NODE ); - TEST_ASSERT( FileNode::GetTypeS() == Node::FILE_NODE ); - - { - #if defined( __WINDOWS__ ) - const CompilerNode * cn = ng.CreateCompilerNode( AStackString<>( "c:\\cl.exe" ) ); - #else - const CompilerNode * cn = ng.CreateCompilerNode( AStackString<>( "/usr/bin/gcc" ) ); - #endif - TEST_ASSERT( cn->GetType() == Node::COMPILER_NODE ); - TEST_ASSERT( AStackString<>( "Compiler" ) == cn->GetTypeName() ); - } - - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateCopyFileNode( AStackString<>( "c:\\dummy" ) ); - #else - const Node * n = ng.CreateCopyFileNode( AStackString<>( "/dummy/dummy" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::COPY_FILE_NODE ); - TEST_ASSERT( CopyFileNode::GetTypeS() == Node::COPY_FILE_NODE ); - TEST_ASSERT( AStackString<>( "CopyFile" ) == n->GetTypeName() ); - } - + // Node names differ on Window vs other platforms due to paths etc #if defined( __WINDOWS__ ) - const DirectoryListNode * dn = ng.CreateDirectoryListNode( AStackString<>( "path\\|*.cpp|false|" ) ); + #define CHOOSE_NAME( WINDOWS_PATH, OTHER_PATH ) AStackString<> name( WINDOWS_PATH ) #else - const DirectoryListNode * dn = ng.CreateDirectoryListNode( AStackString<>( "path/|*.cpp|false|" ) ); + #define CHOOSE_NAME( WINDOWS_PATH, OTHER_PATH ) AStackString<> name( OTHER_PATH ) #endif - TEST_ASSERT( dn->GetType() == Node::DIRECTORY_LIST_NODE ); - TEST_ASSERT( DirectoryListNode::GetTypeS() == Node::DIRECTORY_LIST_NODE ); - TEST_ASSERT( AStackString<>( "Directory" ) == dn->GetTypeName() ); - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateExecNode( AStackString<>( "c:\\execdummy" ) ); - #else - const Node * n = ng.CreateExecNode( AStackString<>( "/execdummy/execdummy" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::EXEC_NODE ); - TEST_ASSERT( ExecNode::GetTypeS() == Node::EXEC_NODE ); - TEST_ASSERT( AStackString<>( "Exec" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateLibraryNode( AStackString<>( "c:\\library.lib" ) ); - #else - const Node * n = ng.CreateLibraryNode( AStackString<>( "/library/library.a" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::LIBRARY_NODE ); - TEST_ASSERT( LibraryNode::GetTypeS() == Node::LIBRARY_NODE ); - TEST_ASSERT( AStackString<>( "Library" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateObjectNode( AStackString<>( "c:\\object.lib" ) ); - #else - const Node * n = ng.CreateObjectNode( AStackString<>( "/library/object.o" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::OBJECT_NODE ); - TEST_ASSERT( ObjectNode::GetTypeS() == Node::OBJECT_NODE ); - TEST_ASSERT( AStackString<>( "Object" ) == n->GetTypeName() ); - } - { - const Node * n = ng.CreateAliasNode( AStackString<>( "alias" ) ); - TEST_ASSERT( n->GetType() == Node::ALIAS_NODE ); - TEST_ASSERT( AliasNode::GetTypeS() == Node::ALIAS_NODE ); - TEST_ASSERT( AStackString<>( "Alias" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - AStackString<> dllName( "c:\\lib.dll" ); - #else - AStackString<> dllName( "/tmp/lib.so" ); - #endif - const Node * n = ng.CreateDLLNode( dllName ); - TEST_ASSERT( n->GetType() == Node::DLL_NODE ); - TEST_ASSERT( DLLNode::GetTypeS() == Node::DLL_NODE ); - TEST_ASSERT( AStackString<>( "DLL" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - AStackString<> exeName( "c:\\exe.exe" ); - #else - AStackString<> exeName( "/tmp/exe.exe" ); - #endif - const Node * n = ng.CreateExeNode( exeName ); - TEST_ASSERT( n->GetType() == Node::EXE_NODE ); - TEST_ASSERT( ExeNode::GetTypeS() == Node::EXE_NODE ); - TEST_ASSERT( AStackString<>( "Exe" ) == n->GetTypeName() ); - } - { - const Node * n = ng.CreateUnityNode( AStackString<>( "Unity" ) ); - TEST_ASSERT( n->GetType() == Node::UNITY_NODE ); - TEST_ASSERT( UnityNode::GetTypeS() == Node::UNITY_NODE ); - TEST_ASSERT( AStackString<>( "Unity" ) == n->GetTypeName() ); - } - { - #if defined( __WINDOWS__ ) - const Node * n = ng.CreateCSNode( AStackString<>( "c:\\csharp.dll" ) ); - #else - const Node * n = ng.CreateCSNode( AStackString<>( "/dummy/csharp.dll" ) ); - #endif - TEST_ASSERT( n->GetType() == Node::CS_NODE); - TEST_ASSERT( CSNode::GetTypeS() == Node::CS_NODE ); - TEST_ASSERT( AStackString<>( "C#" ) == n->GetTypeName() ); - } + // Test each node can be created and type mappings are consistent + #define TEST_NODE( TYPE, TYPE_ENUM, FRIENDLY_TYPE, WINDOWS_PATH, OTHER_PATH ) \ + { \ + CHOOSE_NAME( WINDOWS_PATH, OTHER_PATH ); \ + const TYPE * node = ng.CreateNode( name ); \ + TEST_ASSERT( node->GetType() == Node::TYPE_ENUM ); \ + TEST_ASSERT( TYPE::GetTypeS() == Node::TYPE_ENUM ); \ + TEST_ASSERT( AStackString<>( FRIENDLY_TYPE ) == node->GetTypeName() ); \ + } while( false ) + + // TODO:C - It would be nice to restructure this so that new nodes are automatically tested + TEST_NODE( FileNode, FILE_NODE, "File", "file", "file" ); + TEST_NODE( CompilerNode, COMPILER_NODE, "Compiler", "c:\\cl.exe", "/usr/bin/gcc" ); + TEST_NODE( CopyFileNode, COPY_FILE_NODE, "CopyFile", "c:\\file", "/path/file" ); + TEST_NODE( DirectoryListNode, DIRECTORY_LIST_NODE, "Directory", "path\\|*.cpp|false|", "path/|*.cpp|false|" ); + TEST_NODE( ExecNode, EXEC_NODE, "Exec", "c:\\exec", "/path/exec" ); + TEST_NODE( LibraryNode, LIBRARY_NODE, "Library", "c:\\library.lib", "/library/library.a" ); + TEST_NODE( ObjectNode, OBJECT_NODE, "Object", "c:\\object.obj", "/path/object.a" ); + TEST_NODE( AliasNode, ALIAS_NODE, "Alias", "alias", "alias" ); + TEST_NODE( DLLNode, DLL_NODE, "DLL", "c:\\lib.dll", "/tmp/lib.so" ); + TEST_NODE( ExeNode, EXE_NODE, "Exe", "c:\\exe.exe", "/tmp/exe.exe" ); + TEST_NODE( UnityNode, UNITY_NODE, "Unity", "unity", "unity" ); + TEST_NODE( CSNode, CS_NODE, "C#", "c:\\csharp.dll", "/path/csharp.dll" ); + TEST_NODE( TestNode, TEST_NODE, "Test", "c:\\output.txt", "/path/output.txt" ); + TEST_NODE( RemoveDirNode, REMOVE_DIR_NODE, "RemoveDir", "remove", "remove" ); + + #undef TEST_NODE + #undef CHOOSE_NAME } // FileNode @@ -238,7 +171,7 @@ void TestGraph::SingleFileNode() const TEST_ASSERT( ng.FindNode( testFileName ) == nullptr ); // create the node, and make sure we can access it by name - FileNode * node = ng.CreateFileNode( testFileName ); + FileNode * node = ng.CreateNode( testFileName ); TEST_ASSERT( ng.FindNode( testFileName ) == node ); TEST_ASSERT( fb.Build( node ) ); @@ -257,7 +190,7 @@ void TestGraph::SingleFileNodeMissing() const // make a node for a file that does not exist const AStackString<> testFileName( "ThisFileDoesNotExist.cpp" ); - FileNode * node = ng.CreateFileNode( testFileName ); + FileNode * node = ng.CreateNode( testFileName ); // make sure build still passes // a missing file is not an error. it would need to be required by something @@ -291,7 +224,7 @@ void TestGraph::TestDirectoryListNode() const name ); // create the node, and make sure we can access it by name - DirectoryListNode * node = ng.CreateDirectoryListNode( name ); + DirectoryListNode * node = ng.CreateNode( name ); node->m_Path = testFolder; node->m_Patterns = patterns; const BFFToken * token = nullptr; @@ -1005,4 +938,42 @@ void TestGraph::CyclicDependency() const } } +// DBLocation +//------------------------------------------------------------------------------ +void TestGraph::DBLocation() const +{ + FBuildTestOptions options; + options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/fbuild.bff"; + options.m_SaveDBOnCompletion = true; + + AString dbFileDefaultLocation( "Tools/FBuild/FBuildTest/Data/TestGraph/DatabaseLocation/" ); + AString dbFileExplicitLocation( "../tmp/Test/Graph/DatabaseLocation/GraphDB.fdb" ); + + // Build a target and let serialization save to default location + { + FBuildForTest fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); + + const AString & dbFile( fBuild.GetDependencyGraphFile() ); + EnsureFileDoesNotExist( dbFile ); + + TEST_ASSERT( fBuild.Build( "TestTarget" ) ); + TEST_ASSERT( PathUtils::PathBeginsWith( dbFile, dbFileDefaultLocation ) ); + } + + // Build a target and let serialization save to explicitly specified location + { + options.m_DBFile = dbFileExplicitLocation; + + FBuildForTest fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); + + const AString & dbFile( fBuild.GetDependencyGraphFile() ); + EnsureFileDoesNotExist( dbFile ); + + TEST_ASSERT( fBuild.Build( "TestTarget" ) ); + TEST_ASSERT( PathUtils::ArePathsEqual( dbFile, dbFileExplicitLocation ) ); + } +} + //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp index e06606f0d..3033cf3d4 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp @@ -130,7 +130,7 @@ void TestIf::IfFunctionBool() const // Unary TEST_EXP_TRUE( "", "!false" ); TEST_EXP_FALSE( "", "!true" ); - + // Binary TEST_EXP_TRUE( "", "false != true" ); @@ -182,7 +182,7 @@ void TestIf::IfFunctionInt() const TEST_EXP_TRUE( "", "2 < 3" ); TEST_EXP_TRUE( "", "2 <= 3" ); TEST_EXP_FALSE( "", "2 <= 1" ); - + // Greater Than TEST_EXP_TRUE( "", "2 > 1" ); TEST_EXP_TRUE( "", "2 >= 1" ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp index ce7c0277e..0da2ccf61 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp @@ -170,8 +170,10 @@ class BaseNode : public Node REFLECT_DECLARE( BaseNode ) public: BaseNode() - : Node( AStackString<>( "dummy" ), Node::PROXY_NODE, 0 ) - {} + : Node( Node::PROXY_NODE ) + { + SetName( AStackString<>( "placeholder" ) ); + } virtual bool Initialize( NodeGraph & /*nodeGraph*/, const BFFToken * /*funcStartIter*/, const Function * /*function*/ ) override { ASSERT( false ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp index f58fba393..a86bfd574 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp @@ -1393,7 +1393,7 @@ void TestProjectGeneration::VSExternalProj_ExternalProject() const CheckStatsNode( 1, 1, Node::ALIAS_NODE ); // because of the external module, peek how many of them were actually processed, depending if using the module is - // enforced or not in the actual fbuild.bff + // enforced or not in the actual fbuild.bff const FBuildStats& stats = FBuild::Get().GetStats(); const FBuildStats::Stats& nodeStatsExternal = stats.GetStatsFor( Node::VSPROJEXTERNAL_NODE ); const size_t actualNumExtSeen = nodeStatsExternal.m_NumProcessed; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestRemoveDir.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestRemoveDir.cpp index 434bea54d..a78264fb3 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestRemoveDir.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestRemoveDir.cpp @@ -19,31 +19,15 @@ class TestRemoveDir : public FBuildTest private: DECLARE_TESTS - void CreateNode() const; void RemoveDir() const; }; // Register Tests //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestRemoveDir ) - REGISTER_TEST( CreateNode ) REGISTER_TEST( RemoveDir ) REGISTER_TESTS_END -// CreateNode -//------------------------------------------------------------------------------ -void TestRemoveDir::CreateNode() const -{ - FBuild fb; - NodeGraph ng; - - const RemoveDirNode * removeDirNode = ng.CreateRemoveDirNode( AStackString<>( "name" ) ); - - TEST_ASSERT( removeDirNode->GetType() == Node::REMOVE_DIR_NODE ); - TEST_ASSERT( RemoveDirNode::GetTypeS() == Node::REMOVE_DIR_NODE ); - TEST_ASSERT( AStackString<>( "RemoveDir" ) == removeDirNode->GetTypeName() ); -} - // RemoveDir //------------------------------------------------------------------------------ void TestRemoveDir::RemoveDir() const diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestTest.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestTest.cpp index 3d219c1c4..352e8cbf9 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestTest.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestTest.cpp @@ -20,7 +20,6 @@ class TestTest : public FBuildTest private: DECLARE_TESTS - void CreateNode() const; void Build() const; void Build_NoRebuild() const; void Fail_ReturnCode() const; @@ -32,7 +31,6 @@ class TestTest : public FBuildTest // Register Tests //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestTest ) - REGISTER_TEST( CreateNode ) REGISTER_TEST( Build ) REGISTER_TEST( Build_NoRebuild ) REGISTER_TEST( Fail_ReturnCode ) @@ -41,22 +39,6 @@ REGISTER_TESTS_BEGIN( TestTest ) REGISTER_TEST( Exclusions ) REGISTER_TESTS_END -// CreateNode -//------------------------------------------------------------------------------ -void TestTest::CreateNode() const -{ - FBuild fb; - NodeGraph ng; - - AStackString<> outputPath; - NodeGraph::CleanPath( AStackString<>( "output.txt" ), outputPath ); - const TestNode * testNode = ng.CreateTestNode( outputPath ); - - TEST_ASSERT( testNode->GetType() == Node::TEST_NODE ); - TEST_ASSERT( TestNode::GetTypeS() == Node::TEST_NODE ); - TEST_ASSERT( AStackString<>( "Test" ) == testNode->GetTypeName() ); -} - // Build //------------------------------------------------------------------------------ void TestTest::Build() const diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp index 097dab02e..0ffa693ff 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp @@ -195,7 +195,7 @@ void TestUnity::DetectDeletedUnityFiles() const { // Ensure that a generated Unity file that has been deleted is // detected and regenerated - + EnsureFileDoesNotExist( "../tmp/Test/Unity/Unity1.cpp" ); EnsureFileDoesNotExist( "../tmp/Test/Unity/Unity2.cpp" ); diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff index d7e37a276..cf03fc6e9 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorker.bff @@ -55,7 +55,7 @@ // Output .CompilerOutputPath = '$OutputBase$/$ProjectPath$/' - + #if __WINDOWS__ .CompilerOptions + ' -D"PSAPI_VERSION=1"' #endif @@ -80,12 +80,13 @@ 'Core-Lib-$Platform$-$BuildConfigName$', 'LZ4-Lib-$Platform$-$BuildConfigName$' 'xxHash-Lib-$Platform$-$BuildConfigName$' + 'Zstd-Lib-$Platform$-$BuildConfigName$' } #if __WINDOWS__ + 'FBuildWorker-Res-$Platform$-$BuildConfigName$' #endif #if __OSX__ - + '$ProjectName$-OSXRes-$Platform$-$BuildConfigName$' + + '$ProjectName$-OSXRes-$Platform$-$BuildConfigName$' #endif #if __LINUX__ .LinkerOutput = '$OutputBase$/$ProjectPath$/fbuildworker$ExeExtension$' // NOTE: lower case @@ -135,7 +136,7 @@ ^ProjectConfigs + .ProjectConfig #endif } - + // Create Universal binaries #if CLANG_SUPPORTS_ARMOSX ForEach( .BuildConfig in .BuildConfigs ) @@ -148,7 +149,7 @@ } } #endif - + // Aliases //-------------------------------------------------------------------------- CreateCommonAliases( .ProjectName ) diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp index 3aaf8cfc2..586a820c3 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Worker/Worker.cpp @@ -278,17 +278,17 @@ bool Worker::HasEnoughMemory() return ( m_LastMemoryCheckResult != 0 ); } m_TimerLastMemoryCheck.Start(); - + PERFORMANCE_INFORMATION memInfo; memInfo.cb = sizeof( memInfo ); if ( GetPerformanceInfo( &memInfo, sizeof( memInfo ) ) ) { const uint64_t limitMemSize = memInfo.CommitLimit * memInfo.PageSize; const uint64_t currentMemSize = memInfo.CommitTotal * memInfo.PageSize; - + // Calculate the free memory in MiB. const uint64_t freeMemSize = ( limitMemSize - currentMemSize ) / MEGABYTE; - + // Check if the free memory is high enough const WorkerSettings & ws = WorkerSettings::Get(); if ( freeMemSize > ws.GetMinimumFreeMemoryMiB() ) @@ -297,7 +297,7 @@ bool Worker::HasEnoughMemory() return true; } } - + // The machine doesn't have enough memory or query failed. Exclude this machine from worker pool. m_LastMemoryCheckResult = 0; return false; diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h b/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h index c8a02cfbc..cb1ae6a2f 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h +++ b/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h @@ -50,7 +50,7 @@ class WorkerSettings : public Singleton< WorkerSettings > private: Mode m_Mode; - uint32_t m_IdleThresholdPercent; + uint32_t m_IdleThresholdPercent; uint32_t m_NumCPUsToUse; bool m_StartMinimized; uint64_t m_SettingsWriteTime; // FileTime of settings when last changed/written to disk diff --git a/Code/Tools/FBuild/Integration/notepad++markup.xml b/Code/Tools/FBuild/Integration/notepad++markup.xml index eba899a61..696675c72 100644 --- a/Code/Tools/FBuild/Integration/notepad++markup.xml +++ b/Code/Tools/FBuild/Integration/notepad++markup.xml @@ -25,7 +25,7 @@ Alias CSAssembly Compiler Copy CopyDir DLL Error Exec Executable ForEach If Library ListDependencies ObjectList Print RemoveDir Settings Test TextFile Unity Using VCXProject VSProjectExternal VSSolution XCodeProject - AdditionalOptions AdditionalSymbolSearchPaths AllowCaching AllowDistribution AllowResponseFile ApplicationEnvironment ApplicationType ApplicationTypeRevision AssemblySearchPath AumidOverride BaseProjectConfig BaseSolutionConfig BuildLogFile CachePath CachePathMountPoint CachePluginDLL CachePluginDLLConfig ClangFixupUnity_Disable ClangGCCUpdateXLanguageArg ClangRewriteIncludes Compiler CompilerFamily CompilerForceUsing CompilerInputAllowNoFiles CompilerInputExcludePath CompilerInputExcludePattern CompilerInputExcludedFiles CompilerInputFile CompilerInputFiles CompilerInputFilesRoot CompilerInputObjectLists CompilerInputPath CompilerInputPathRecurse CompilerInputPattern CompilerInputUnity CompilerOptions CompilerOptionsDeoptimized CompilerOutput CompilerOutputExtension CompilerOutputKeepBaseExtension CompilerOutputPath CompilerOutputPrefix CompilerReferences Condition Config CustomEnvironmentVariables DebuggerFlavor DefaultLanguage DeoptimizeWritableFiles DeoptimizeWritableFilesWithToken Dependencies DeploymentFiles DeploymentType Dest DistributableJobMemoryLimitMiB Environment ExecAlways ExecAlwaysShowOutput ExecArguments ExecExecutable ExecInput ExecInputExcludePath ExecInputExcludePattern ExecInputExcludedFiles ExecInputPath ExecInputPathRecurse ExecInputPattern ExecOutput ExecReturnCode ExecUseStdOutAsOutput ExecWorkingDir Executable ExecutableRootPath ExternalProjectPath ExtraFiles FileType ForceResponseFile ForcedIncludes ForcedUsingAssemblies Hidden IncludeSearchPath IntermediateDirectory Items Keyword LayoutDir LayoutExtensionFilter Librarian LibrarianAdditionalInputs LibrarianAllowResponseFile LibrarianForceResponseFile LibrarianOptions LibrarianOutput LibrarianType Libraries Libraries2 Linker LinkerAllowResponseFile LinkerAssemblyResources LinkerForceResponseFile LinkerLinkObjects LinkerOptions LinkerOutput LinkerStampExe LinkerStampExeArgs LinkerType LinuxProjectType LocalDebuggerCommand LocalDebuggerCommandArguments LocalDebuggerEnvironment LocalDebuggerWorkingDirectory Output OutputDirectory PCHInputFile PCHObjectFileName PCHOptions PCHOutputFile PackagePath Path Pattern Patterns Platform PlatformToolset PreBuildDependencies Preprocessor PreprocessorDefinitions PreprocessorOptions Project ProjectAllowedFileExtensions ProjectBasePath ProjectBuildCommand ProjectCleanCommand ProjectConfigs ProjectFileTypes ProjectFiles ProjectFilesToExclude ProjectGuid ProjectInputPaths ProjectInputPathsExclude ProjectInputPathsRecurse ProjectOutput ProjectPatternToExclude ProjectProjectImports ProjectProjectReferences ProjectRebuildCommand ProjectReferences ProjectSccEntrySAK ProjectTypeGuid Projects RemoteDebuggerCommand RemoteDebuggerCommandArguments RemoteDebuggerWorkingDirectory RemoveExcludeFiles RemoveExcludePaths RemovePaths RemovePathsRecurse RemovePatterns RootNamespace SimpleDistributionMode SolutionBuildProject SolutionConfig SolutionConfigs SolutionDependencies SolutionDeployProjects SolutionFolders SolutionMinimumVisualStudioVersion SolutionOutput SolutionPlatform SolutionProjects SolutionVisualStudioVersion Source SourceExcludePaths SourceMapping_Experimental SourcePaths SourcePathsPattern SourcePathsRecurse Target TargetLinuxPlatform Targets TestAlwaysShowOutput TestArguments TestExecutable TestInput TestInputExcludePath TestInputExcludePattern TestInputExcludedFiles TestInputPath TestInputPathRecurse TestInputPattern TestOutput TestTimeOut TestWorkingDir TextFileAlways TextFileInputStrings TextFileOutput UnityInputExcludePath UnityInputExcludePattern UnityInputExcludedFiles UnityInputFiles UnityInputIsolateListFile UnityInputIsolateWritableFiles UnityInputIsolateWritableFilesLimit UnityInputIsolatedFiles UnityInputObjectLists UnityInputPath UnityInputPathRecurse UnityInputPattern UnityNumFiles UnityOutputPath UnityOutputPattern UnityPCH UseLightCache_Experimental UseRelativePaths_Experimental VS2012EnumBugFix WorkerConnectionLimit Workers XCodeBaseSDK XCodeBuildToolArgs XCodeBuildToolPath XCodeBuildWorkingDir XCodeCommandLineArguments XCodeCommandLineArgumentsDisabled XCodeDebugWorkingDir XCodeDocumentVersioning XCodeIphoneOSDeploymentTarget XCodeOrganizationName Xbox360DebuggerCommand + AdditionalOptions AdditionalSymbolSearchPaths AllowCaching AllowDistribution AllowResponseFile AndroidApkLocation ApplicationEnvironment ApplicationType ApplicationTypeRevision AssemblySearchPath AumidOverride BaseProjectConfig BaseSolutionConfig BuildLogFile CachePath CachePathMountPoint CachePluginDLL CachePluginDLLConfig ClangFixupUnity_Disable ClangGCCUpdateXLanguageArg ClangRewriteIncludes Compiler CompilerFamily CompilerForceUsing CompilerInputAllowNoFiles CompilerInputExcludePath CompilerInputExcludePattern CompilerInputExcludedFiles CompilerInputFile CompilerInputFiles CompilerInputFilesRoot CompilerInputObjectLists CompilerInputPath CompilerInputPathRecurse CompilerInputPattern CompilerInputUnity CompilerOptions CompilerOptionsDeoptimized CompilerOutput CompilerOutputExtension CompilerOutputKeepBaseExtension CompilerOutputPath CompilerOutputPrefix CompilerReferences Condition Config CustomEnvironmentVariables DebuggerFlavor DefaultLanguage DeoptimizeWritableFiles DeoptimizeWritableFilesWithToken Dependencies DeploymentFiles DeploymentType Dest DistributableJobMemoryLimitMiB Environment ExecAlways ExecAlwaysShowOutput ExecArguments ExecExecutable ExecInput ExecInputExcludePath ExecInputExcludePattern ExecInputExcludedFiles ExecInputPath ExecInputPathRecurse ExecInputPattern ExecOutput ExecReturnCode ExecUseStdOutAsOutput ExecWorkingDir Executable ExecutableRootPath ExternalProjectPath ExtraFiles FileType ForceResponseFile ForcedIncludes ForcedUsingAssemblies Hidden IncludeSearchPath IntermediateDirectory Items Keyword LayoutDir LayoutExtensionFilter Librarian LibrarianAdditionalInputs LibrarianAllowResponseFile LibrarianForceResponseFile LibrarianOptions LibrarianOutput LibrarianType Libraries Libraries2 Linker LinkerAllowResponseFile LinkerAssemblyResources LinkerForceResponseFile LinkerLinkObjects LinkerOptions LinkerOutput LinkerStampExe LinkerStampExeArgs LinkerType LinuxProjectType LocalDebuggerCommand LocalDebuggerCommandArguments LocalDebuggerEnvironment LocalDebuggerWorkingDirectory Output OutputDirectory PCHInputFile PCHObjectFileName PCHOptions PCHOutputFile PackagePath Path Pattern Patterns Platform PlatformToolset PreBuildDependencies Preprocessor PreprocessorDefinitions PreprocessorOptions Project ProjectAllowedFileExtensions ProjectBasePath ProjectBuildCommand ProjectCleanCommand ProjectConfigs ProjectFileTypes ProjectFiles ProjectFilesToExclude ProjectGuid ProjectInputPaths ProjectInputPathsExclude ProjectInputPathsRecurse ProjectOutput ProjectPatternToExclude ProjectProjectImports ProjectProjectReferences ProjectRebuildCommand ProjectReferences ProjectSccEntrySAK ProjectTypeGuid Projects RemoteDebuggerCommand RemoteDebuggerCommandArguments RemoteDebuggerWorkingDirectory RemoveExcludeFiles RemoveExcludePaths RemovePaths RemovePathsRecurse RemovePatterns RootNamespace SimpleDistributionMode SolutionBuildProject SolutionConfig SolutionConfigs SolutionDependencies SolutionDeployProjects SolutionFolders SolutionMinimumVisualStudioVersion SolutionOutput SolutionPlatform SolutionProjects SolutionVisualStudioVersion Source SourceExcludePaths SourceMapping_Experimental SourcePaths SourcePathsPattern SourcePathsRecurse Target TargetLinuxPlatform Targets TestAlwaysShowOutput TestArguments TestExecutable TestInput TestInputExcludePath TestInputExcludePattern TestInputExcludedFiles TestInputPath TestInputPathRecurse TestInputPattern TestOutput TestTimeOut TestWorkingDir TextFileAlways TextFileInputStrings TextFileOutput UnityInputExcludePath UnityInputExcludePattern UnityInputExcludedFiles UnityInputFiles UnityInputIsolateListFile UnityInputIsolateWritableFiles UnityInputIsolateWritableFilesLimit UnityInputIsolatedFiles UnityInputObjectLists UnityInputPath UnityInputPathRecurse UnityInputPattern UnityNumFiles UnityOutputPath UnityOutputPattern UnityPCH UseLightCache_Experimental UseRelativePaths_Experimental VS2012EnumBugFix WorkerConnectionLimit Workers XCodeBaseSDK XCodeBuildToolArgs XCodeBuildToolPath XCodeBuildWorkingDir XCodeCommandLineArguments XCodeCommandLineArgumentsDisabled XCodeDebugWorkingDir XCodeDocumentVersioning XCodeIphoneOSDeploymentTarget XCodeOrganizationName Xbox360DebuggerCommand ) %1 %2 %3 diff --git a/Code/Tools/FBuild/Integration/usertype.dat b/Code/Tools/FBuild/Integration/usertype.dat index 4fa7d62c7..aaa4dbb4d 100644 --- a/Code/Tools/FBuild/Integration/usertype.dat +++ b/Code/Tools/FBuild/Integration/usertype.dat @@ -45,6 +45,7 @@ AdditionalSymbolSearchPaths AllowCaching AllowDistribution AllowResponseFile +AndroidApkLocation ApplicationEnvironment ApplicationType ApplicationTypeRevision diff --git a/Code/fbuild.bff b/Code/fbuild.bff index d29eef84e..cbeb7b1c7 100644 --- a/Code/fbuild.bff +++ b/Code/fbuild.bff @@ -400,6 +400,7 @@ Settings // External #include "..\External\LZ4\LZ4.bff" #include "../External/xxHash/xxHash.bff" +#include "../External/Zstd/Zstd.bff" // Test Framework #include "TestFramework\TestFramework.bff" @@ -425,7 +426,7 @@ ForEach( .BuildConfig in .BuildConfigs ) Using( .BuildConfig ) Alias( 'All-$Platform$-$BuildConfigName$' ) { .Targets = .'Targets_$Platform$_$BuildConfigName$' } - + // Create additional Universal targets #if CLANG_SUPPORTS_ARMOSX If( .Platform == 'ARMOSX' ) @@ -603,7 +604,7 @@ Alias( 'All+Tests' ) ForEach( .BuildConfig in .BuildConfigs ) { Using( .BuildConfig ) - .ProjectConfig = [ + .ProjectConfig = [ Using( .'Project_$Platform$_$BuildConfigName$' ) .ProjectBuildCommand = 'cd ^$(SolutionDir)\..\..\Code\ & fbuild solution -vs' .ProjectRebuildCommand = 'cd ^$(SolutionDir)\..\..\Code\ & fbuild solution -vs -clean' @@ -657,7 +658,7 @@ Alias( 'All+Tests' ) .Folder_External = [ .Path = 'External' - .Projects = { 'LZ4-proj', 'SDKs-proj', 'xxHash-proj' } + .Projects = { 'LZ4-proj', 'SDKs-proj', 'xxHash-proj', 'Zstd-proj' } ] .Folder_Test = [ @@ -698,7 +699,8 @@ Alias( 'All+Tests' ) 'LZ4-xcodeproj' 'OSUI-xcodeproj' 'TestFramework-xcodeproj' - 'xxHash-xcodeproj' } + 'xxHash-xcodeproj' + 'Zstd-xcodeproj' } .ProjectConfigs = {} ForEach( .BuildConfig in .BuildConfigs ) { diff --git a/External/LZ4/LZ4.bff b/External/LZ4/LZ4.bff index 9f9ec1558..36cb9dc70 100644 --- a/External/LZ4/LZ4.bff +++ b/External/LZ4/LZ4.bff @@ -23,7 +23,7 @@ // When mixing old versions of Clang with VS2019, there are some problems with certian // bitscan intrinsics, so we disable their use here - #if USING_CLANG_7 || USING_CLANG_8 || USING_CLANG_9 + #if USING_CLANG_8 || USING_CLANG_9 .LZ4CompilerOptions + ' -DLZ4_FORCE_SW_BITCOUNT' #endif ] diff --git a/External/SDK/Clang/Clang.bff b/External/SDK/Clang/Clang.bff index 3cc75c65d..86f15a586 100644 --- a/External/SDK/Clang/Clang.bff +++ b/External/SDK/Clang/Clang.bff @@ -7,13 +7,13 @@ //#define USING_CLANG_3 //#define USING_CLANG_6 #define USING_CLANG_10 + //#define USING_CLANG_14 #endif #if __OSX__ && !CI_BUILD //#define USING_CLANG_8 #define USING_CLANG_12 #endif #if __WINDOWS__ && !CI_BUILD - //#define USING_CLANG_7 //#define USING_CLANG_8 //#define USING_CLANG_9 //#define USING_CLANG_10 @@ -37,6 +37,9 @@ #if USING_CLANG_10 #include "Linux/Clang10.bff" #endif + #if USING_CLANG_14 + #include "Linux/Clang14.bff" + #endif #endif #if __OSX__ #if CI_BUILD @@ -53,9 +56,6 @@ #if CI_BUILD #define USING_CLANG_11 #endif - #if USING_CLANG_7 - #include "Windows/Clang7.bff" - #endif #if USING_CLANG_8 #include "Windows/Clang8.bff" #endif diff --git a/External/SDK/Clang/Linux/Clang14.bff b/External/SDK/Clang/Linux/Clang14.bff new file mode 100644 index 000000000..81ef2e505 --- /dev/null +++ b/External/SDK/Clang/Linux/Clang14.bff @@ -0,0 +1,63 @@ +// Clang 14.0.0 +//------------------------------------------------------------------------------ +.Clang14_BasePath = '/usr/lib/llvm-14/bin' + +// Compiler +//------------------------------------------------------------------------------ +Compiler( 'Compiler-Clang14' ) +{ + .Root = '$Clang14_BasePath$' + .Executable = '$Root$/clang' + + // Allow tests to activate some experimental behavior + #if ENABLE_RELATIVE_PATHS + .UseRelativePaths_Experimental = true + #endif + #if ENABLE_SOURCE_MAPPING + .SourceMapping_Experimental = '/fastbuild-test-mapping' + #endif +} + +// ToolChain +//------------------------------------------------------------------------------ +.ToolChain_Clang_Linux = +[ + .Platform = 'x64ClangLinux' + + // Compiler Options + .Compiler = 'Compiler-Clang14' + .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output + + ' -c' // Compile only + + ' -g' // Generate debug info + + ' -m64' // x86_64 + + ' -D__LINUX__' // Platform define + + // Include paths + + ' -I./' + + // Enable warnings + + ' -Wall -Werror -Wfatal-errors' // warnings as errors + + ' -Wextra' + + ' -Wshadow' + + .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' + + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types + .CompilerOptionsC = ' -x c $CommonCompilerOptions$' + + // Librarian + .Librarian = '$Clang14_BasePath$/llvm-ar' + .LibrarianOptions = 'rcs "%2" "%1"' + + // Linker + .Linker = '$Clang14_BasePath$/clang++' + .LinkerOptions = '"%1" -o "%2"' + + // File Extensions + .LibExtension = '.a' + .ExeExtension = '' + + // Exception Control + .UseExceptions = ' -fexceptions' +] + +//------------------------------------------------------------------------------ diff --git a/External/SDK/Clang/Windows/Clang7.bff b/External/SDK/Clang/Windows/Clang7.bff deleted file mode 100644 index ff33573df..000000000 --- a/External/SDK/Clang/Windows/Clang7.bff +++ /dev/null @@ -1,160 +0,0 @@ -// Clang 7.x.x -//------------------------------------------------------------------------------ -.Clang7_BasePath = '$_CURRENT_BFF_DIR_$/7.0.1' - -// Compiler -//------------------------------------------------------------------------------ -Compiler( 'Compiler-Clang7' ) -{ - .Root = '$Clang7_BasePath$' - .Executable = '$Root$\bin\clang-cl.exe' - - // Allow tests to activate some experimental behavior - #if ENABLE_RELATIVE_PATHS - .UseRelativePaths_Experimental = true - #endif - #if ENABLE_SOURCE_MAPPING - .SourceMapping_Experimental = '/fastbuild-test-mapping' - #endif -} - -// Compiler -//------------------------------------------------------------------------------ -Compiler( 'Compiler-Clang7-NonCL' ) -{ - .Root = '$Clang7_BasePath$' - .Executable = '$Root$\bin\clang.exe' - - // Allow tests to activate some experimental behavior - #if ENABLE_RELATIVE_PATHS - .UseRelativePaths_Experimental = true - #endif - #if ENABLE_SOURCE_MAPPING - .SourceMapping_Experimental = '/fastbuild-test-mapping' - #endif -} - -// ToolChain -//------------------------------------------------------------------------------ -.ToolChain_Clang_Windows_Common = -[ - // Clang for Windows relies on the VS being present: - // - crt headers - // - crt libs/dlls - Using( .ToolChain_VS_Windows_X64 ) - - .Platform = 'x64Clang' - - // Librarian - .Librarian = '$Clang7_BasePath$\bin\llvm-ar.exe' - .LibrarianOptions = 'rc "%2" "%1"' // NOTE: output must come first - - // Linker - .Linker = '$Clang7_BasePath$\bin\lld-link.exe' - .LinkerOptions = '/NODEFAULTLIB /WX /NOLOGO /INCREMENTAL:NO /OUT:"%2" "%1" /DEBUG' - + .VSLibPaths - - // File Extensions - .LibExtension = '.a' - .ExeExtension = '.exe' -] - -// ToolChain -//------------------------------------------------------------------------------ -.ToolChain_Clang_Windows = -[ - Using( .ToolChain_Clang_Windows_Common ) - - // Compiler Options - .Compiler = 'Compiler-Clang7' - .CommonCompilerOptions = ' -c' // Compile only - + ' /Z7' // Include debug info - - // Include paths - + ' -I"./"' - + .VSIncludePaths_ClangCl - - // x64 - + ' -m64' - - // Enable warnings - + ' -Wall -Werror -Wfatal-errors' // warnings as errors - + ' -Wextra' - + ' -Wshadow' - - // No RTTI - + ' /GR-' - - // Warnings that are not useful - + ' -Wno-#pragma-messages' // warning : %s [-W#pragma-messages] - + ' -Wno-c++98-compat-pedantic' // variadic macros are incompatible with C++98 - + ' -Wno-exit-time-destructors' // declaration requires an exit-time destructor - + ' -Wno-global-constructors' // declaration requires a global destructor - + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types - + ' -Wno-missing-prototypes' // no previous prototype for function '%s' - + ' -Wno-missing-variable-declarations' // no previous extern declaration for non-static variable '%s' - - // Warnings that fire but might be best to be fixed - + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier - + ' -Wno-deprecated' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor - + ' -Wno-missing-noreturn' // function '%s' could be declared with attribute 'noreturn' - + ' -Wno-old-style-cast' // use of old-style cast - + ' -Wno-switch-enum' // %u enumeration values not explicitly handled in switch: '%s', '%s'... - - .CompilerOptions = ' /TP -o"%2" "%1" $CommonCompilerOptions$' - + ' /std:c++17' // Enable c++17 features - .CompilerOptionsC = ' /TC -o"%2" "%1" $CommonCompilerOptions$' - .PCHOptions = ' /TP $CommonCompilerOptions$ "%1" /Fo"%3" /Fp"%2" /Yc"PrecompiledHeader.h"' - + ' /std:c++17' // Enable c++17 features - - // Exception Control - .UseExceptions = ' /EHs' -] - -// ToolChain -//------------------------------------------------------------------------------ -.ToolChain_ClangNonCL_Windows = -[ - Using( .ToolChain_Clang_Windows_Common ) - - // Compiler Options - .Compiler = 'Compiler-Clang7-NonCL' - .CommonCompilerOptions = ' -c' // Compile only - + ' -g' // Include debug info - - // Include paths - + ' "-I./"' - - // x64 - + ' -m64' - - // Enable warnings - + ' -Wall -Werror -Wfatal-errors' // warnings as errors - + ' -Wextra' - + ' -Wshadow' - - // No RTTI - + ' -fno-rtti' - - // Warnings that are not useful - + ' -Wno-#pragma-messages' // warning : %s [-W#pragma-messages] - + ' -Wno-c++98-compat-pedantic' // variadic macros are incompatible with C++98 - + ' -Wno-exit-time-destructors' // declaration requires an exit-time destructor - + ' -Wno-global-constructors' // declaration requires a global destructor - + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types - + ' -Wno-missing-prototypes' // no previous prototype for function '%s' - + ' -Wno-missing-variable-declarations' // no previous extern declaration for non-static variable '%s' - - // Warnings that fire but might be best to be fixed - + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier - + ' -Wno-deprecated' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor - + ' -Wno-missing-noreturn' // function '%s' could be declared with attribute 'noreturn' - + ' -Wno-old-style-cast' // use of old-style cast - + ' -Wno-switch-enum' // %u enumeration values not explicitly handled in switch: '%s', '%s'... - - .CompilerOptions = ' -x c++ -o"%2" "%1" $CommonCompilerOptions$' - + ' -std=c++17' // Enable c++17 features - .CompilerOptionsC = ' -x c -o"%2" "%1" $CommonCompilerOptions$' -] - -//------------------------------------------------------------------------------ diff --git a/External/SDK/GCC/GCC.bff b/External/SDK/GCC/GCC.bff index 0d8b6aebc..67c7eeebd 100644 --- a/External/SDK/GCC/GCC.bff +++ b/External/SDK/GCC/GCC.bff @@ -7,6 +7,7 @@ //#define USING_GCC_4 //#define USING_GCC_7 #define USING_GCC_9 + //#define USING_GCC_11 #endif // Activate @@ -24,6 +25,9 @@ #if USING_GCC_9 #include "Linux/GCC9.bff" #endif + #if USING_GCC_11 + #include "Linux/GCC11.bff" + #endif #endif //------------------------------------------------------------------------------ diff --git a/External/SDK/GCC/Linux/GCC11.bff b/External/SDK/GCC/Linux/GCC11.bff new file mode 100644 index 000000000..f132d3e55 --- /dev/null +++ b/External/SDK/GCC/Linux/GCC11.bff @@ -0,0 +1,66 @@ +// GCC 11.3.0 +//------------------------------------------------------------------------------ +.GCC11_BasePath = '/usr/bin' + +// Compiler +//------------------------------------------------------------------------------ +Compiler( 'Compiler-GCC11' ) +{ + .Executable = '$GCC11_BasePath$/x86_64-linux-gnu-gcc-11' + .ExtraFiles = { + '/usr/bin/as' + '/usr/lib/gcc/x86_64-linux-gnu/11/cc1' + '/usr/lib/gcc/x86_64-linux-gnu/11/cc1plus' + } + .CompilerFamily = 'gcc' // TODO: Remove when FASTBuild detection is improved + + // Allow tests to activate some experimental behavior + #if ENABLE_SOURCE_MAPPING + .SourceMapping_Experimental = '/fastbuild-test-mapping' + #endif +} + +// ToolChain +//------------------------------------------------------------------------------ +.ToolChain_GCC_Linux = +[ + .Platform = 'x64Linux' + + // Compiler Options + .Compiler = 'Compiler-GCC11' + .CommonCompilerOptions = ' -o "%2" "%1"' // Input/Output + + ' -c' // Compile only + + ' -g' // Generate debug info + + ' -m64' // x86_64 + + ' -D__LINUX__' // Platform define + + ' -ffreestanding' // prevent extra magic includes like stdc-predefs.h + + // Include paths + + ' -I./' + + // Enable warnings + + ' -Wall -Werror -Wfatal-errors' // warnings as errors + + ' -Wextra' + + ' -Wshadow' + + .CompilerOptions = ' -std=c++11 $CommonCompilerOptions$' + + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types + .CompilerOptionsC = ' -x c $CommonCompilerOptions$' + + // Librarian + .Librarian = '$GCC11_BasePath$/x86_64-linux-gnu-ar' + .LibrarianOptions = 'rcs "%2" "%1"' + + // Linker + .Linker = '$GCC11_BasePath$/x86_64-linux-gnu-g++-11' + .LinkerOptions = '"%1" -o "%2"' + + // File Extensions + .LibExtension = '.a' + .ExeExtension = '' + + // Exception Control + .UseExceptions = ' -fexceptions' +] + +//------------------------------------------------------------------------------ diff --git a/External/Zstd/Zstd.bff b/External/Zstd/Zstd.bff new file mode 100644 index 000000000..6cc807851 --- /dev/null +++ b/External/Zstd/Zstd.bff @@ -0,0 +1,147 @@ +// Zstd +//------------------------------------------------------------------------------ +.ZstdBasePath = '../External/Zstd/zstd-1.5.5/lib' +.ZstdIncludePaths = ' "-I$ZstdBasePath$"' +{ + .ProjectName = 'Zstd' + .ProjectPath = '$ZstdBasePath$' + + // Target/Compiler specific options + .ZstdOptions_x64 = [ + .ZstdCompilerOptions = ' /wd4365' // conversion from '%s' to '%s', signed/unsigned mismatch + + ' /wd4464' // relative include path contains '..' + + ' /wd4574' // '__has_attribute' is defined to be '0': did you mean to use '#if __has_attribute'? + + ' /wd6011' // Dereferencing NULL pointer 'ptr'. + + ' /wd6239' // ( && ) always evaluates to the result of . Did you intend to use the bitwise-and operator? + + ' /wd6293' // Ill-defined for-loop: counts down from minimum. + + ' /wd6326' // Potential comparison of a constant with another constant. + + ' /wd26448' // Consider using gsl::finally if final action is intended (gsl.util). + + ' /wd26462' // The value pointed to by '%s' is assigned only once, mark it as a pointer to const + + ' /wd26818' // Switch statement does not cover all cases. Consider adding a 'default' label (es.79). + + ' /wd26819' // Unannotated fallthrough between switch labels (es.78). + + ' /wd28251' // Inconsistent annotation for '_setjmp': this instance has no annotations. See (0). + + + ' -O2' // Compile with optimizations even in debug to improve performance + ] + .ZstdOptions_x64Clang = [ + .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + .ZstdOptions_x64Linux = [ + .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + + // Disable inline asm - TODO:B Is this a significant perf gain for Linux vs other platforms + // that don't seem to use it? + + ' -DZSTD_DISABLE_ASM' + ] + .ZstdOptions_x64ClangLinux = .ZstdOptions_x64Linux + .ZstdOptions_x64OSX = [ + .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + + // Disable inline asm - TODO:B Is this a significant perf gain for Linux vs other platforms + // that don't seem to use it? + + ' -DZSTD_DISABLE_ASM' + ] + .ZstdOptions_ARMOSX = [ + .ZstdCompilerOptions = ' -O2' // Compile with optimizations even in debug to improve performance + ] + + // Library + //-------------------------------------------------------------------------- + .ProjectConfigs = {} + ForEach( .BuildConfig in .BuildConfigs ) + { + Using( .BuildConfig ) + + .OutputBase + '\$Platform$-$BuildConfigName$' + + Using( ."ZstdOptions_$Platform$" ) + + Unity( '$ProjectName$-Unity-$Platform$-$BuildConfigName$' ) + { + .UnityInputPattern = '*.c' + .UnityInputPath = { + '$ZstdBasePath$/common/' + '$ZstdBasePath$/compress/' + '$ZstdBasePath$/decompress/' + } + .UnityInputExcludedFiles = { + '$ZstdBasePath$/common/debug.c' + '$ZstdBasePath$/common/threading.c' + '$ZstdBasePath$/compress/zstdmt_compress.c' + } + .UnityNumFiles = 3 + .UnityOutputPath = '$OutputBase$/External/$ProjectName$/' + } + + // Static Library + ObjectList( '$ProjectName$-Lib-$Platform$-$BuildConfigName$' ) + { + .CompilerInputUnity = '$ProjectName$-Unity-$Platform$-$BuildConfigName$' + + // Options + .CompilerOptions = .CompilerOptionsC + + .ZstdIncludePaths + + .ZstdCompilerOptions + + #if __WINDOWS__ + // Remove flags that disable opimizations + - ' /Od' + - ' /RTC1' + + // Disable clang-cl static analysis if enabled for this config + // (we won't fix warnings in 3rd party code) + - ' --analyze' + #else + - ' -O0' + #endif + + // Disable warnings if using Clang. There are too many warnings in Zstd + // and they differ with every version of Clang + - ' -Wall' + - ' -Werror' + - ' -Wfatal-errors' + - ' -Wextra' + - ' -Wshadow' + - ' -Weverything' + + // Output + .CompilerOutputPath = '$OutputBase$/External/$ProjectName$/' + } + Alias( '$ProjectName$-$Platform$-$BuildConfigName$' ) { .Targets = '$ProjectName$-Lib-$Platform$-$BuildConfigName$' } + + #if __WINDOWS__ + .ProjectConfig = [ Using( .'Project_$Platform$_$BuildConfigName$' ) .Target = '$ProjectName$-$Platform$-$BuildConfigName$' ] + ^ProjectConfigs + .ProjectConfig + #endif + #if __OSX__ + .ProjectConfig = [ .Config = '$BuildConfigName$' .Target = '$ProjectName$-x64OSX-$BuildConfigName$' ] + ^ProjectConfigs + .ProjectConfig + #endif + } + + // Aliases + //-------------------------------------------------------------------------- + CreateCommonAliases( .ProjectName ) + + // Visual Studio Project Generation + //-------------------------------------------------------------------------- + #if __WINDOWS__ + .ExtraOptions = [ + .ProjectFiles = '../External/Zstd/Zstd.bff' + ] + CreateVCXProject_Lib( .ProjectName, .ProjectPath, .ProjectConfigs, .ExtraOptions ) + #endif + + // XCode Project Generation + //-------------------------------------------------------------------------- + #if __OSX__ + XCodeProject( '$ProjectName$-xcodeproj' ) + { + .ProjectOutput = '../tmp/XCode/Projects/0_External/$ProjectName$.xcodeproj/project.pbxproj' + .ProjectInputPaths = '$ProjectPath$/' + .ProjectBasePath = '$ProjectPath$/' + + .XCodeBuildWorkingDir = '../../../../Code/' + } + #endif +} diff --git a/External/Zstd/zstd-1.5.5/CHANGELOG b/External/Zstd/zstd-1.5.5/CHANGELOG new file mode 100644 index 000000000..c7a7506ee --- /dev/null +++ b/External/Zstd/zstd-1.5.5/CHANGELOG @@ -0,0 +1,800 @@ +v1.5.5 (Apr 2023) +fix: fix rare corruption bug affecting the high compression mode, reported by @danlark1 (#3517, @terrelln) +perf: improve mid-level compression speed (#3529, #3533, #3543, @yoniko and #3552, @terrelln) +lib: deprecated bufferless block-level API (#3534) by @terrelln +cli: mmap large dictionaries to save memory, by @daniellerozenblit +cli: improve speed of --patch-from mode (~+50%) (#3545) by @daniellerozenblit +cli: improve i/o speed (~+10%) when processing lots of small files (#3479) by @felixhandte +cli: zstd no longer crashes when requested to write into write-protected directory (#3541) by @felixhandte +cli: fix decompression into block device using -o, reported by @georgmu (#3583) +build: fix zstd CLI compiled with lzma support but not zlib support (#3494) by @Hello71 +build: fix cmake does no longer require 3.18 as minimum version (#3510) by @kou +build: fix MSVC+ClangCL linking issue (#3569) by @tru +build: fix zstd-dll, version of zstd CLI that links to the dynamic library (#3496) by @yoniko +build: fix MSVC warnings (#3495) by @embg +doc: updated zstd specification to clarify corner cases, by @Cyan4973 +doc: document how to create fat binaries for macos (#3568) by @rickmark +misc: improve seekable format ingestion speed (~+100%) for very small chunk sizes (#3544) by @Cyan4973 +misc: tests/fullbench can benchmark multiple files (#3516) by @dloidolt + +v1.5.4 (Feb 2023) +perf: +20% faster huffman decompression for targets that can't compile x64 assembly (#3449, @terrelln) +perf: up to +10% faster streaming compression at levels 1-2 (#3114, @embg) +perf: +4-13% for levels 5-12 by optimizing function generation (#3295, @terrelln) +pref: +3-11% compression speed for `arm` target (#3199, #3164, #3145, #3141, #3138, @JunHe77 and #3139, #3160, @danlark1) +perf: +5-30% faster dictionary compression at levels 1-4 (#3086, #3114, #3152, @embg) +perf: +10-20% cold dict compression speed by prefetching CDict tables (#3177, @embg) +perf: +1% faster compression by removing a branch in ZSTD_fast_noDict (#3129, @felixhandte) +perf: Small compression ratio improvements in high compression mode (#2983, #3391, @Cyan4973 and #3285, #3302, @daniellerozenblit) +perf: small speed improvement by better detecting `STATIC_BMI2` for `clang` (#3080, @TocarIP) +perf: Improved streaming performance when `ZSTD_c_stableInBuffer` is set (#2974, @Cyan4973) +cli: Asynchronous I/O for improved cli speed (#2975, #2985, #3021, #3022, @yoniko) +cli: Change `zstdless` behavior to align with `zless` (#2909, @binhdvo) +cli: Keep original file if `-c` or `--stdout` is given (#3052, @dirkmueller) +cli: Keep original files when result is concatenated into a single output with `-o` (#3450, @Cyan4973) +cli: Preserve Permissions and Ownership of regular files (#3432, @felixhandte) +cli: Print zlib/lz4/lzma library versions with `-vv` (#3030, @terrelln) +cli: Print checksum value for single frame files with `-lv` (#3332, @Cyan4973) +cli: Print `dictID` when present with `-lv` (#3184, @htnhan) +cli: when `stderr` is *not* the console, disable status updates, but preserve final summary (#3458, @Cyan4973) +cli: support `--best` and `--no-name` in `gzip` compatibility mode (#3059, @dirkmueller) +cli: support for `posix` high resolution timer `clock_gettime()`, for improved benchmark accuracy (#3423, @Cyan4973) +cli: improved help/usage (`-h`, `-H`) formatting (#3094, @dirkmueller and #3385, @jonpalmisc) +cli: Fix better handling of bogus numeric values (#3268, @ctkhanhly) +cli: Fix input consists of multiple files _and_ `stdin` (#3222, @yoniko) +cli: Fix tiny files passthrough (#3215, @cgbur) +cli: Fix for `-r` on empty directory (#3027, @brailovich) +cli: Fix empty string as argument for `--output-dir-*` (#3220, @embg) +cli: Fix decompression memory usage reported by `-vv --long` (#3042, @u1f35c, and #3232, @zengyijing) +cli: Fix infinite loop when empty input is passed to trainer (#3081, @terrelln) +cli: Fix `--adapt` doesn't work when `--no-progress` is also set (#3354, @terrelln) +api: Support for Block-Level Sequence Producer (#3333, @embg) +api: Support for in-place decompression (#3432, @terrelln) +api: New `ZSTD_CCtx_setCParams()` function, set all parameters defined in a `ZSTD_compressionParameters` structure (#3403, @Cyan4973) +api: Streaming decompression detects incorrect header ID sooner (#3175, @Cyan4973) +api: Window size resizing optimization for edge case (#3345, @daniellerozenblit) +api: More accurate error codes for busy-loop scenarios (#3413, #3455, @Cyan4973) +api: Fix limit overflow in `compressBound` and `decompressBound` (#3362, #3373, Cyan4973) reported by @nigeltao +api: Deprecate several advanced experimental functions: streaming (#3408, @embg), copy (#3196, @mileshu) +bug: Fix corruption that rarely occurs in 32-bit mode with wlog=25 (#3361, @terrelln) +bug: Fix for block-splitter (#3033, @Cyan4973) +bug: Fixes for Sequence Compression API (#3023, #3040, @Cyan4973) +bug: Fix leaking thread handles on Windows (#3147, @animalize) +bug: Fix timing issues with cmake/meson builds (#3166, #3167, #3170, @Cyan4973) +build: Allow user to select legacy level for cmake (#3050, @shadchin) +build: Enable legacy support by default in cmake (#3079, @niamster) +build: Meson build script improvements (#3039, #3120, #3122, #3327, #3357, @eli-schwartz and #3276, @neheb) +build: Add aarch64 to supported architectures for zstd_trace (#3054, @ooosssososos) +build: support AIX architecture (#3219, @qiongsiwu) +build: Fix `ZSTD_LIB_MINIFY` build macro, which now reduces static library size by half (#3366, @terrelln) +build: Fix Windows issues with Multithreading translation layer (#3364, #3380, @yoniko) and ARM64 target (#3320, @cwoffenden) +build: Fix `cmake` script (#3382, #3392, @terrelln and #3252 @Tachi107 and #3167 @Cyan4973) +doc: Updated man page, providing more details for `--train` mode (#3112, @Cyan4973) +doc: Add decompressor errata document (#3092, @terrelln) +misc: Enable Intel CET (#2992, #2994, @hjl-tools) +misc: Fix `contrib/` seekable format (#3058, @yhoogstrate and #3346, @daniellerozenblit) +misc: Improve speed of the one-file library generator (#3241, @wahern and #3005, @cwoffenden) + +v1.5.3 (dev version, unpublished) + +v1.5.2 (Jan, 2022) +perf: Regain Minimal memset()-ing During Reuse of Compression Contexts (@Cyan4973, #2969) +build: Build Zstd with `noexecstack` on All Architectures (@felixhandte, #2964) +doc: Clarify Licensing (@terrelln, #2981) + +v1.5.1 (Dec, 2021) +perf: rebalanced compression levels, to better match the intended speed/level curve, by @senhuang42 +perf: faster huffman decoder, using x64 assembly, by @terrelln +perf: slightly faster high speed modes (strategies fast & dfast), by @felixhandte +perf: improved binary size and faster compilation times, by @terrelln +perf: new row64 mode, used notably in level 12, by @senhuang42 +perf: faster mid-level compression speed in presence of highly repetitive patterns, by @senhuang42 +perf: minor compression ratio improvements for small data at high levels, by @cyan4973 +perf: reduced stack usage (mostly useful for Linux Kernel), by @terrelln +perf: faster compression speed on incompressible data, by @bindhvo +perf: on-demand reduced ZSTD_DCtx state size, using build macro ZSTD_DECODER_INTERNAL_BUFFER, at a small cost of performance, by @bindhvo +build: allows hiding static symbols in the dynamic library, using build macro, by @skitt +build: support for m68k (Motorola 68000's), by @cyan4973 +build: improved AIX support, by @Helflym +build: improved meson unofficial build, by @eli-schwartz +cli : custom memory limit when training dictionary (#2925), by @embg +cli : report advanced parameters information when compressing in very verbose mode (``-vv`), by @Svetlitski-FB + +v1.5.0 (May 11, 2021) +api: Various functions promoted from experimental to stable API: (#2579-2581, @senhuang42) + `ZSTD_defaultCLevel()` + `ZSTD_getDictID_fromCDict()` +api: Several experimental functions have been deprecated and will emit a compiler warning (#2582, @senhuang42) + `ZSTD_compress_advanced()` + `ZSTD_compress_usingCDict_advanced()` + `ZSTD_compressBegin_advanced()` + `ZSTD_compressBegin_usingCDict_advanced()` + `ZSTD_initCStream_srcSize()` + `ZSTD_initCStream_usingDict()` + `ZSTD_initCStream_usingCDict()` + `ZSTD_initCStream_advanced()` + `ZSTD_initCStream_usingCDict_advanced()` + `ZSTD_resetCStream()` +api: ZSTDMT_NBWORKERS_MAX reduced to 64 for 32-bit environments (@Cyan4973) +perf: Significant speed improvements for middle compression levels (#2494, @senhuang42 @terrelln) +perf: Block splitter to improve compression ratio, enabled by default for high compression levels (#2447, @senhuang42) +perf: Decompression loop refactor, speed improvements on `clang` and for `--long` modes (#2614 #2630, @Cyan4973) +perf: Reduced stack usage during compression and decompression entropy stage (#2522 #2524, @terrelln) +bug: Improve setting permissions of created files (#2525, @felixhandte) +bug: Fix large dictionary non-determinism (#2607, @terrelln) +bug: Fix non-determinism test failures on Linux i686 (#2606, @terrelln) +bug: Fix various dedicated dictionary search bugs (#2540 #2586, @senhuang42 @felixhandte) +bug: Ensure `ZSTD_estimateCCtxSize*() `monotonically increases with compression level (#2538, @senhuang42) +bug: Fix --patch-from mode parameter bound bug with small files (#2637, @occivink) +bug: Fix UBSAN error in decompression (#2625, @terrelln) +bug: Fix superblock compression divide by zero bug (#2592, @senhuang42) +bug: Make the number of physical CPU cores detection more robust (#2517, @PaulBone) +doc: Improve `zdict.h` dictionary training API documentation (#2622, @terrelln) +doc: Note that public `ZSTD_free*()` functions accept NULL pointers (#2521, @animalize) +doc: Add style guide docs for open source contributors (#2626, @Cyan4973) +tests: Better regression test coverage for different dictionary modes (#2559, @senhuang42) +tests: Better test coverage of index reduction (#2603, @terrelln) +tests: OSS-Fuzz coverage for seekable format (#2617, @senhuang42) +tests: Test coverage for ZSTD threadpool API (#2604, @senhuang42) +build: Dynamic library built multithreaded by default (#2584, @senhuang42) +build: Move `zstd_errors.h` and `zdict.h` to `lib/` root (#2597, @terrelln) +build: Allow `ZSTDMT_JOBSIZE_MIN` to be configured at compile-time, reduce default to 512KB (#2611, @Cyan4973) +build: Single file library build script moved to `build/` directory (#2618, @felixhandte) +build: `ZBUFF_*()` is no longer built by default (#2583, @senhuang42) +build: Fixed Meson build (#2548, @SupervisedThinking @kloczek) +build: Fix excessive compiler warnings with clang-cl and CMake (#2600, @nickhutchinson) +build: Detect presence of `md5` on Darwin (#2609, @felixhandte) +build: Avoid SIGBUS on armv6 (#2633, @bmwiedmann) +cli: `--progress` flag added to always display progress bar (#2595, @senhuang42) +cli: Allow reading from block devices with `--force` (#2613, @felixhandte) +cli: Fix CLI filesize display bug (#2550, @Cyan4973) +cli: Fix windows CLI `--filelist` end-of-line bug (#2620, @Cyan4973) +contrib: Various fixes for linux kernel patch (#2539, @terrelln) +contrib: Seekable format - Decompression hanging edge case fix (#2516, @senhuang42) +contrib: Seekable format - New seek table-only API (#2113 #2518, @mdittmer @Cyan4973) +contrib: Seekable format - Fix seek table descriptor check when loading (#2534, @foxeng) +contrib: Seekable format - Decompression fix for large offsets, (#2594, @azat) +misc: Automatically published release tarballs available on Github (#2535, @felixhandte) + +v1.4.9 (Mar 1, 2021) +bug: Use `umask()` to Constrain Created File Permissions (#2495, @felixhandte) +bug: Make Simple Single-Pass Functions Ignore Advanced Parameters (#2498, @terrelln) +api: Add (De)Compression Tracing Functionality (#2482, @terrelln) +api: Support References to Multiple DDicts (#2446, @senhuang42) +api: Add Function to Generate Skippable Frame (#2439, @senhuang42) +perf: New Algorithms for the Long Distance Matcher (#2483, @mpu) +perf: Performance Improvements for Long Distance Matcher (#2464, @mpu) +perf: Don't Shrink Window Log when Streaming with a Dictionary (#2451, @terrelln) +cli: Fix `--output-dir-mirror`'s Rejection of `..`-Containing Paths (#2512, @felixhandte) +cli: Allow Input From Console When `-f`/`--force` is Passed (#2466, @felixhandte) +cli: Improve Help Message (#2500, @senhuang42) +tests: Remove Flaky Tests (#2455, #2486, #2445, @Cyan4973) +tests: Correctly Invoke md5 Utility on NetBSD (#2492, @niacat) +tests: Avoid Using `stat -c` on NetBSD (#2513, @felixhandte) +build: Zstd CLI Can Now be Linked to Dynamic `libzstd` (#2457, #2454 @Cyan4973) +build: Hide and Avoid Using Static-Only Symbols (#2501, #2504, @skitt) +build: CMake: Enable Only C for lib/ and programs/ Projects (#2498, @concatime) +build: CMake: Use `configure_file()` to Create the `.pc` File (#2462, @lazka) +build: Fix Fuzzer Compiler Detection & Update UBSAN Flags (#2503, @terrelln) +build: Add Guards for `_LARGEFILE_SOURCE` and `_LARGEFILE64_SOURCE` (#2444, @indygreg) +build: Improve `zlibwrapper` Makefile (#2437, @Cyan4973) +contrib: Add `recover_directory` Program (#2473, @terrelln) +doc: Change License Year to 2021 (#2452 & #2465, @terrelln & @senhuang42) +doc: Fix Typos (#2459, @ThomasWaldmann) + +v1.4.8 (Dec 18, 2020) +hotfix: wrong alignment of an internal buffer + +v1.4.7 (Dec 16, 2020) +perf: stronger --long mode at high compression levels, by @senhuang42 +perf: stronger --patch-from at high compression levels, thanks to --long improvements +perf: faster dictionary compression at medium compression levels, by @felixhandte +perf: small speed & memory usage improvements for ZSTD_compress2(), by @terrelln +perf: improved fast compression speeds with Visual Studio, by @animalize +cli : Set nb of threads with environment variable ZSTD_NBTHREADS, by @senhuang42 +cli : accept decompressing files with *.zstd suffix +cli : provide a condensed summary by default when processing multiple files +cli : fix : stdin input no longer confused as user prompt +cli : improve accuracy of several error messages +api : new sequence ingestion API, by @senhuang42 +api : shared thread pool: control total nb of threads used by multiple compression jobs, by @marxin +api : new ZSTD_getDictID_fromCDict(), by @LuAPi +api : zlibWrapper only uses public API, and is compatible with dynamic library, by @terrelln +api : fix : multithreaded compression has predictable output even in special cases (see #2327) (issue not accessible from cli) +api : fix : dictionary compression correctly respects dictionary compression level (see #2303) (issue not accessible from cli) +build: fix cmake script when using path with spaces, by @terrelln +build: improved compile-time detection of aarch64/neon platforms, by @bsdimp +build: Fix building on AIX 5.1, by @likema +build: compile paramgrill with cmake on Windows, requested by @mirh +doc : clarify repcode updates in format specification, by @felixhandte + +v1.4.6 +fix : Always return dstSize_tooSmall when that is the case +fix : Fix ZSTD_initCStream_advanced() with static allocation and no dictionary +perf: Improve small block decompression speed by 20%+, by @terrelln +perf: Reduce compression stack usage by 1 KB, by @terrelln +perf: Improve decompression speed by improving ZSTD_wildcopy, by @helloguo (#2252, #2256) +perf: Improve histogram construction, by @cyan4973 (#2253) +cli : Add --output-dir-mirror option, by @xxie24 (#2219) +cli : Warn when (de)compressing multiple files into a single output, by @senhuang42 (#2279) +cli : Improved progress bar and status summary when (de)compressing multiple files, by @senhuang42 (#2283) +cli : Call stat less often, by @felixhandte (#2262) +cli : Allow --patch-from XXX and --filelist XXX in addition to --patch-from=XXX and --filelist=XXX, by @cyan4973 (#2250) +cli : Allow --patch-from to compress stdin with --stream-size, by @bimbashrestha (#2206) +api : Do not install zbuff.h, since it has long been deprecated, by @cyan4973 (#2166). +api : Fix ZSTD_CCtx_setParameter() with ZSTD_c_compressionLevel to make 0 mean default level, by @i-do-cpp (#2291) +api : Rename ZSTDMT_NBTHREADS_MAX to ZSTDMT_NBWORKERS_MAX, by @marxin (#2228). +build: Install pkg-config file with CMake and MinGW, by @tonytheodore (#2183) +build: Install DLL with CMake on Windows, by @BioDataAnalysis (#2221) +build: Fix DLL install location with CMake, by @xantares and @bimbashrestha (#2186) +build: Add ZSTD_NO_UNUSED_FUNCTIONS macro to hide unused functions +build: Add ZSTD_NO_INTRINSICS macro to avoid explicit intrinsics +build: Add STATIC_BMI2 macro for compile time detection of BMI2 on MSVC, by @Niadb (#2258) +build: Fix -Wcomma warnings, by @cwoffenden +build: Remove distutils requirement for meson build, by @neheb (#2197) +build: Fix cli compilation with uclibc +build: Fix cli compilation without st_mtime, by @ffontaine (#2246) +build: Fix shadowing warnings in library +build: Fix single file library compilation with Enscripten, by @yoshihitoh (#2227) +misc: Improve single file library and include dictBuilder, by @cwoffenden +misc: Allow compression dictionaries with missing symbols +misc: Add freestanding translation script in contrib/freestanding_lib +misc: Collect all of zstd's libc dependencies into zstd_deps.h +doc : Add ZSTD_versionString() to manual, by @animalize +doc : Fix documentation for ZSTD_CCtxParams_setParameter(), by @felixhandte (#2270) + +v1.4.5 (May 22, 2020) +fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln +perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln +perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta) +perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh) +perf: Small level 1 compression speed gains (depending on compiler) +cli : New --patch-from command, create and apply patches from files, by @bimbashreshta +cli : New --filelist= : Provide a list of files to operate upon from a file +cli : -b -d command can now benchmark decompression on multiple files +cli : New --no-content-size command +cli : New --show-default-cparams information command +api : ZDICT_finalizeDictionary() is promoted to stable (#2111) +api : new experimental parameter ZSTD_d_stableOutBuffer (#2094) +build: Generate a single-file libzstd library (#2065, by @cwoffenden) +build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte) +build: zstd now compiles cleanly under -pedantic (#2099) +build: zstd now compiles with make-4.3 +build: Support mingw cross-compilation from Linux, by @Ericson2314 +build: Meson multi-thread build fix on windows +build: Some misc icc fixes backed by new ci test on travis +misc: bitflip analyzer tool, by @felixhandte +misc: Extend largeNbDicts benchmark to compression +misc: Edit-distance match finder in contrib/ +doc : Improved beginner CONTRIBUTING.md docs +doc : New issue templates for zstd + +v1.4.4 (Nov 6, 2019) +perf: Improved decompression speed, by > 10%, by @terrelln +perf: Better compression speed when re-using a context, by @felixhandte +perf: Fix compression ratio when compressing large files with small dictionary, by @senhuang42 +perf: zstd reference encoder can generate RLE blocks, by @bimbashrestha +perf: minor generic speed optimization, by @davidbolvansky +api: new ability to extract sequences from the parser for analysis, by @bimbashrestha +api: fixed decoding of magic-less frames, by @terrelln +api: fixed ZSTD_initCStream_advanced() performance with fast modes, reported by @QrczakMK +cli: Named pipes support, by @bimbashrestha +cli: short tar's extension support, by @stokito +cli: command --output-dir-flat= , generates target files into requested directory, by @senhuang42 +cli: commands --stream-size=# and --size-hint=#, by @nmagerko +cli: command --exclude-compressed, by @shashank0791 +cli: faster `-t` test mode +cli: improved some error messages, by @vangyzen +cli: fix command `-D dictionary` on Windows, reported by @artyompetrov +cli: fix rare deadlock condition within dictionary builder, by @terrelln +build: single-file decoder with emscripten compilation script, by @cwoffenden +build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive +build: fixed deprecation warning for certain gcc version, reported by @jasonma163 +build: fix compilation on old gcc versions, by @cemeyer +build: improved installation directories for cmake script, by Dmitri Shubin +pack: modified pkgconfig, for better integration into openwrt, requested by @neheb +misc: Improved documentation : ZSTD_CLEVEL, DYNAMIC_BMI2, ZSTD_CDict, function deprecation, zstd format +misc: fixed educational decoder : accept larger literals section, and removed UNALIGNED() macro + +v1.4.3 (Aug 20, 2019) +bug: Fix Dictionary Compression Ratio Regression by @cyan4973 (#1709) +bug: Fix Buffer Overflow in legacy v0.3 decompression by @felixhandte (#1722) +build: Add support for IAR C/C++ Compiler for Arm by @joseph0918 (#1705) + +v1.4.2 (Jul 26, 2019) +bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696) +bug: Fix seekable decompression in-memory API by @iburinoc (#1695) +misc: Validate blocks are smaller than size limit by @vivekmg (#1685) +misc: Restructure source files by @ephiepark (#1679) + +v1.4.1 (Jul 20, 2019) +bug: Fix data corruption in niche use cases by @terrelln (#1659) +bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595) +bug: Fix out of bounds read by @terrelln (#1590) +perf: Improve decode speed by ~7% @mgrice (#1668) +perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681) +perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658) +perf: Improve compression ratio for small windowLog by @cyan4973 (#1624) +perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635) +api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656) +cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640) +cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631) +cli: Restrict read permissions on destination files by @chungy (#1644) +cli: zstdgrep: handle -f flag by @felixhandte (#1618) +cli: zstdcat: follow symlinks by @vejnar (#1604) +doc: Remove extra size limit on compressed blocks by @felixhandte (#1689) +doc: Fix typo by @yk-tanigawa (#1633) +doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629) +build: CMake: support building with LZ4 @leeyoung624 (#1626) +build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647) +build: CMake: respect existing uninstall target by @j301scott (#1619) +build: Make: skip multithread tests when built without support by @michaelforney (#1620) +build: Make: Fix examples/ test target by @sjnam (#1603) +build: Meson: rename options out of deprecated namespace by @lzutao (#1665) +build: Meson: fix build by @lzutao (#1602) +build: Visual Studio: don't export symbols in static lib by @scharan (#1650) +build: Visual Studio: fix linking by @absotively (#1639) +build: Fix MinGW-W64 build by @myzhang1029 (#1600) +misc: Expand decodecorpus coverage by @ephiepark (#1664) + +v1.4.0 (Apr 17, 2019) +perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln +api: Move the advanced API, including all functions in the staging section, to the stable section +api: Make ZSTD_e_flush and ZSTD_e_end block for maximum forward progress +api: Rename ZSTD_CCtxParam_getParameter to ZSTD_CCtxParams_getParameter +api: Rename ZSTD_CCtxParam_setParameter to ZSTD_CCtxParams_setParameter +api: Don't export ZSTDMT functions from the shared library by default +api: Require ZSTD_MULTITHREAD to be defined to use ZSTDMT +api: Add ZSTD_decompressBound() to provide an upper bound on decompressed size by @shakeelrao +api: Fix ZSTD_decompressDCtx() corner cases with a dictionary +api: Move ZSTD_getDictID_*() functions to the stable section +api: Add ZSTD_c_literalCompressionMode flag to enable or disable literal compression by @terrelln +api: Allow compression parameters to be set when a dictionary is used +api: Allow setting parameters before or after ZSTD_CCtx_loadDictionary() is called +api: Fix ZSTD_estimateCStreamSize_usingCCtxParams() +api: Setting ZSTD_d_maxWindowLog to 0 means use the default +cli: Ensure that a dictionary is not used to compress itself by @shakeelrao +cli: Add --[no-]compress-literals flag to enable or disable literal compression +doc: Update the examples to use the advanced API +doc: Explain how to transition from old streaming functions to the advanced API in the header +build: Improve the Windows release packages +build: Improve CMake build by @hjmjohnson +build: Build fixes for FreeBSD by @lwhsu +build: Remove redundant warnings by @thatsafunnyname +build: Fix tests on OpenBSD by @bket +build: Extend fuzzer build system to work with the new clang engine +build: CMake now creates the libzstd.so.1 symlink +build: Improve Menson build by @lzutao +misc: Fix symbolic link detection on FreeBSD +misc: Use physical core count for -T0 on FreeBSD by @cemeyer +misc: Fix zstd --list on truncated files by @kostmo +misc: Improve logging in debug mode by @felixhandte +misc: Add CirrusCI tests by @lwhsu +misc: Optimize dictionary memory usage in corner cases +misc: Improve the dictionary builder on small or homogeneous data +misc: Fix spelling across the repo by @jsoref + +v1.3.8 (Dec 28, 2018) +perf: better decompression speed on large files (+7%) and cold dictionaries (+15%) +perf: slightly better compression ratio at high compression modes +api : finalized advanced API, last stage before "stable" status +api : new --rsyncable mode, by @terrelln +api : support decompression of empty frames into NULL (used to be an error) (#1385) +build: new set of macros to build a minimal size decoder, by @felixhandte +build: fix compilation on MIPS32, reported by @clbr (#1441) +build: fix compilation with multiple -arch flags, by @ryandesign +build: highly upgraded meson build, by @lzutao +build: improved buck support, by @obelisk +build: fix cmake script : can create debug build, by @pitrou +build: Makefile : grep works on both colored consoles and systems without color support +build: fixed zstd-pgo, by @bmwiedemann +cli : support ZSTD_CLEVEL environment variable, by @yijinfb (#1423) +cli : --no-progress flag, preserving final summary (#1371), by @terrelln +cli : ensure destination file is not source file (#1422) +cli : clearer error messages, especially when input file not present +doc : clarified zstd_compression_format.md, by @ulikunitz +misc: fixed zstdgrep, returns 1 on failure, by @lzutao +misc: NEWS renamed as CHANGELOG, in accordance with fboss + +v1.3.7 (Oct 20, 2018) +perf: slightly better decompression speed on clang (depending on hardware target) +fix : performance of dictionary compression for small input < 4 KB at levels 9 and 10 +build: no longer build backtrace by default in release mode; restrict further automatic mode +build: control backtrace support through build macro BACKTRACE +misc: added man pages for zstdless and zstdgrep, by @samrussell + +v1.3.6 (Oct 6, 2018) +perf: much faster dictionary builder, by @jenniferliu +perf: faster dictionary compression on small data when using multiple contexts, by @felixhandte +perf: faster dictionary decompression when using a very large number of dictionaries simultaneously +cli : fix : does no longer overwrite destination when source does not exist (#1082) +cli : new command --adapt, for automatic compression level adaptation +api : fix : block api can be streamed with > 4 GB, reported by @catid +api : reduced ZSTD_DDict size by 2 KB +api : minimum negative compression level is defined, and can be queried using ZSTD_minCLevel(). +build: support Haiku target, by @korli +build: Read Legacy format is limited to v0.5+ by default. Can be changed at compile time with macro ZSTD_LEGACY_SUPPORT. +doc : zstd_compression_format.md updated to match wording in IETF RFC 8478 +misc: tests/paramgrill, a parameter optimizer, by @GeorgeLu97 + +v1.3.5 (Jun 29, 2018) +perf: much faster dictionary compression, by @felixhandte +perf: small quality improvement for dictionary generation, by @terrelln +perf: slightly improved high compression levels (notably level 19) +mem : automatic memory release for long duration contexts +cli : fix : overlapLog can be manually set +cli : fix : decoding invalid lz4 frames +api : fix : performance degradation for dictionary compression when using advanced API, by @terrelln +api : change : clarify ZSTD_CCtx_reset() vs ZSTD_CCtx_resetParameters(), by @terrelln +build: select custom libzstd scope through control macros, by @GeorgeLu97 +build: OpenBSD patch, by @bket +build: make and make all are compatible with -j +doc : clarify zstd_compression_format.md, updated for IETF RFC process +misc: pzstd compatible with reproducible compilation, by @lamby + +v1.3.4 (Mar 27, 2018) +perf: faster speed (especially decoding speed) on recent cpus (haswell+) +perf: much better performance associating --long with multi-threading, by @terrelln +perf: better compression at levels 13-15 +cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior) +cli : smoother status report in multi-threading mode +cli : added command --fast=#, for faster compression modes +cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb) +api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode +api : compression levels can be negative, for even more speed +api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime +api : ZSTDMT can accept new compression parameters during compression +api : implemented all advanced dictionary decompression prototypes +build: improved meson recipe, by Shawn Landden (@shawnl) +build: VS2017 scripts, by @HaydnTrigg +misc: all /contrib projects fixed +misc: added /contrib/docker script by @gyscos + +v1.3.3 (Dec 21, 2017) +perf: faster zstd_opt strategy (levels 16-19) +fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen +cli : fix : content size written in header by default +cli : fix : improved LZ4 format support, by @felixhandte +cli : new : hidden command `-S`, to benchmark multiple files while generating one result per file +api : fix : support large skippable frames, by @terrelln +api : fix : streaming interface was adding a useless 3-bytes null block to small frames +api : change : when setting `pledgedSrcSize`, use `ZSTD_CONTENTSIZE_UNKNOWN` macro value to mean "unknown" +build: fix : compilation under rhel6 and centos6, reported by @pixelb +build: added `check` target + +v1.3.2 (Oct 10, 2017) +new : long range mode, using --long command, by Stella Lau (@stellamplau) +new : ability to generate and decode magicless frames (#591) +changed : maximum nb of threads reduced to 200, to avoid address space exhaustion in 32-bits mode +fix : multi-threading compression works with custom allocators +fix : ZSTD_sizeof_CStream() was over-evaluating memory usage +fix : a rare compression bug when compression generates very large distances and bunch of other conditions (only possible at --ultra -22) +fix : 32-bits build can now decode large offsets (levels 21+) +cli : added LZ4 frame support by default, by Felix Handte (@felixhandte) +cli : improved --list output +cli : new : can split input file for dictionary training, using command -B# +cli : new : clean operation artefact on Ctrl-C interruption +cli : fix : do not change /dev/null permissions when using command -t with root access, reported by @mike155 (#851) +cli : fix : write file size in header in multiple-files mode +api : added macro ZSTD_COMPRESSBOUND() for static allocation +api : experimental : new advanced decompression API +api : fix : sizeof_CCtx() used to over-estimate +build: fix : no-multithread variant compiles without pool.c dependency, reported by Mitchell Blank Jr (@mitchblank) (#819) +build: better compatibility with reproducible builds, by Bernhard M. Wiedemann (@bmwiedemann) (#818) +example : added streaming_memory_usage +license : changed /examples license to BSD + GPLv2 +license : fix a few header files to reflect new license (#825) + +v1.3.1 (Aug 21, 2017) +New license : BSD + GPLv2 +perf: substantially decreased memory usage in Multi-threading mode, thanks to reports by Tino Reichardt (@mcmilk) +perf: Multi-threading supports up to 256 threads. Cap at 256 when more are requested (#760) +cli : improved and fixed --list command, by @ib (#772) +cli : command -vV to list supported formats, by @ib (#771) +build : fixed binary variants, reported by @svenha (#788) +build : fix Visual compilation for non x86/x64 targets, reported by Greg Slazinski (@GregSlazinski) (#718) +API exp : breaking change : ZSTD_getframeHeader() provides more information +API exp : breaking change : pinned down values of error codes +doc : fixed huffman example, by Ulrich Kunitz (@ulikunitz) +new : contrib/adaptive-compression, I/O driven compression strength, by Paul Cruz (@paulcruz74) +new : contrib/long_distance_matching, statistics by Stella Lau (@stellamplau) +updated : contrib/linux-kernel, by Nick Terrell (@terrelln) + +v1.3.0 (Jul 6, 2017) +cli : new : `--list` command, by Paul Cruz +cli : changed : xz/lzma support enabled by default +cli : changed : `-t *` continue processing list after a decompression error +API : added : ZSTD_versionString() +API : promoted to stable status : ZSTD_getFrameContentSize(), by Sean Purcell +API exp : new advanced API : ZSTD_compress_generic(), ZSTD_CCtx_setParameter() +API exp : new : API for static or external allocation : ZSTD_initStatic?Ctx() +API exp : added : ZSTD_decompressBegin_usingDDict(), requested by Guy Riddle (#700) +API exp : clarified memory estimation / measurement functions. +API exp : changed : strongest strategy renamed ZSTD_btultra, fastest strategy ZSTD_fast set to 1 +tools : decodecorpus can generate random dictionary-compressed samples, by Paul Cruz +new : contrib/seekable_format, demo and API, by Sean Purcell +changed : contrib/linux-kernel, updated version and license, by Nick Terrell + +v1.2.0 (May 5, 2017) +cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable) +cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell +cli : new : zstdmt symlink hardwired to `zstd -T0` +cli : new : command --threads=# (#671) +cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell +cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters +cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell +cli : fix : does not output compressed data on console +cli : fix : ignore symbolic links unless --force specified, +API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument +API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters. +API : improved: ZSTDMT_compressCCtx() reduced memory usage +API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634) +API : fix : src size stored in frame header is controlled at end of frame +API : fix : enforced consistent rules for pledgedSrcSize==0 (#641) +API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate +build: improved cmake script, by @Majlen +build: enabled Multi-threading support for *BSD, by Baptiste Daroussin +tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target. +new : contrib/linux-kernel version, by Nick Terrell + +v1.1.4 (Mar 18, 2017) +cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski +cli : new : advanced benchmark command --priority=rt +cli : fix : write on sparse-enabled file systems in 32-bits mode, by @ds77 +cli : fix : --rm remains silent when input is stdin +cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski +speed : improved decompression speed in streaming mode for single shot scenarios (+5%) +memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB +arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell +API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize() +API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value) +build : new: meson build system in contrib/meson, by Dima Krasner +build : improved cmake script, by @Majlen +build : added -Wformat-security flag, as recommended by Padraig Brady +doc : new : educational decoder, by Sean Purcell + +v1.1.3 (Feb 7, 2017) +cli : zstd can decompress .gz files (can be disabled with `make zstd-nogz` or `make HAVE_ZLIB=0`) +cli : new : experimental target `make zstdmt`, with multi-threading support +cli : new : improved dictionary builder "cover" (experimental), by Nick Terrell, based on prior work by Giuseppe Ottaviano. +cli : new : advanced commands for detailed parameters, by Przemyslaw Skibinski +cli : fix zstdless on Mac OS-X, by Andrew Janke +cli : fix #232 "compress non-files" +dictBuilder : improved dictionary generation quality, thanks to Nick Terrell +API : new : lib/compress/ZSTDMT_compress.h multithreading API (experimental) +API : new : ZSTD_create?Dict_byReference(), requested by Bartosz Taudul +API : new : ZDICT_finalizeDictionary() +API : fix : ZSTD_initCStream_usingCDict() properly writes dictID into frame header, by Gregory Szorc (#511) +API : fix : all symbols properly exposed in libzstd, by Nick Terrell +build : support for Solaris target, by Przemyslaw Skibinski +doc : clarified specification, by Sean Purcell + +v1.1.2 (Dec 15, 2016) +API : streaming : decompression : changed : automatic implicit reset when chain-decoding new frames without init +API : experimental : added : dictID retrieval functions, and ZSTD_initCStream_srcSize() +API : zbuff : changed : prototypes now generate deprecation warnings +lib : improved : faster decompression speed at ultra compression settings and 32-bits mode +lib : changed : only public ZSTD_ symbols are now exposed +lib : changed : reduced usage of stack memory +lib : fixed : several corner case bugs, by Nick Terrell +cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski +cli : new : preserve file attributes +cli : new : added zstdless and zstdgrep tools +cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd) +cli : fixed : zstdcat +zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski +install : better compatibility with FreeBSD, by Dimitry Andric +source tree : changed : zbuff source files moved to lib/deprecated + +v1.1.1 (Nov 2, 2016) +New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption +New : doc/zstd_manual.html, by Przemyslaw Skibinski +Improved : slightly better compression ratio at --ultra levels (>= 20) +Improved : better memory usage when using streaming compression API, thanks to @Rogier-5 report +Added : API : ZSTD_initCStream_usingCDict(), ZSTD_initDStream_usingDDict() (experimental section) +Added : example/multiple_streaming_compression.c +Changed : zstd_errors.h is now installed within /include (and replaces errors_public.h) +Updated man page +Fixed : zstd-small, zstd-compress and zstd-decompress compilation targets + +v1.1.0 (Sep 28, 2016) +New : contrib/pzstd, parallel version of zstd, by Nick Terrell +added : NetBSD install target (#338) +Improved : speed for batches of small files +Improved : speed of zlib wrapper, by Przemyslaw Skibinski +Changed : libzstd on Windows supports legacy formats, by Christophe Chevalier +Fixed : CLI -d output to stdout by default when input is stdin (#322) +Fixed : CLI correctly detects console on Mac OS-X +Fixed : CLI supports recursive mode `-r` on Mac OS-X +Fixed : Legacy decoders use unified error codes, reported by benrg (#341), fixed by Przemyslaw Skibinski +Fixed : compatibility with OpenBSD, reported by Juan Francisco Cantero Hurtado (#319) +Fixed : compatibility with Hurd, by Przemyslaw Skibinski (#365) +Fixed : zstd-pgo, reported by octoploid (#329) + +v1.0.0 (Sep 1, 2016) +Change Licensing, all project is now BSD, Copyright Facebook +Small decompression speed improvement +API : Streaming API supports legacy format +API : ZDICT_getDictID(), ZSTD_sizeof_{CCtx, DCtx, CStream, DStream}(), ZSTD_setDStreamParameter() +CLI supports legacy formats v0.4+ +Fixed : compression fails on certain huge files, reported by Jesse McGrew +Enhanced documentation, by Przemyslaw Skibinski + +v0.8.1 (Aug 18, 2016) +New streaming API +Changed : --ultra now enables levels beyond 19 +Changed : -i# now selects benchmark time in second +Fixed : ZSTD_compress* can now compress > 4 GB in a single pass, reported by Nick Terrell +Fixed : speed regression on specific patterns (#272) +Fixed : support for Z_SYNC_FLUSH, by Dmitry Krot (#291) +Fixed : ICC compilation, by Przemyslaw Skibinski + +v0.8.0 (Aug 2, 2016) +Improved : better speed on clang and gcc -O2, thanks to Eric Biggers +New : Build on FreeBSD and DragonFly, thanks to JrMarino +Changed : modified API : ZSTD_compressEnd() +Fixed : legacy mode with ZSTD_HEAPMODE=0, by Christopher Bergqvist +Fixed : premature end of frame when zero-sized raw block, reported by Eric Biggers +Fixed : large dictionaries (> 384 KB), reported by Ilona Papava +Fixed : checksum correctly checked in single-pass mode +Fixed : combined --test amd --rm, reported by Andreas M. Nilsson +Modified : minor compression level adaptations +Updated : compression format specification to v0.2.0 +changed : zstd.h moved to /lib directory + +v0.7.5 (Aug 1, 2016) +Transition version, supporting decoding of v0.8.x + +v0.7.4 (Jul 17, 2016) +Added : homebrew for Mac, by Daniel Cade +Added : more examples +Fixed : segfault when using small dictionaries, reported by Felix Handte +Modified : default compression level for CLI is now 3 +Updated : specification, to v0.1.1 + +v0.7.3 (Jul 9, 2016) +New : compression format specification +New : `--` separator, stating that all following arguments are file names. Suggested by Chip Turner. +New : `ZSTD_getDecompressedSize()` +New : OpenBSD target, by Juan Francisco Cantero Hurtado +New : `examples` directory +fixed : dictBuilder using HC levels, reported by Bartosz Taudul +fixed : legacy support from ZSTD_decompress_usingDDict(), reported by Felix Handte +fixed : multi-blocks decoding with intermediate uncompressed blocks, reported by Greg Slazinski +modified : removed "mem.h" and "error_public.h" dependencies from "zstd.h" (experimental section) +modified : legacy functions no longer need magic number + +v0.7.2 (Jul 4, 2016) +fixed : ZSTD_decompressBlock() using multiple consecutive blocks. Reported by Greg Slazinski. +fixed : potential segfault on very large files (many gigabytes). Reported by Chip Turner. +fixed : CLI displays system error message when destination file cannot be created (#231). Reported by Chip Turner. + +v0.7.1 (Jun 23, 2016) +fixed : ZBUFF_compressEnd() called multiple times with too small `dst` buffer, reported by Christophe Chevalier +fixed : dictBuilder fails if first sample is too small, reported by Руслан Ковалёв +fixed : corruption issue, reported by cj +modified : checksum enabled by default in command line mode + +v0.7.0 (Jun 17, 2016) +New : Support for directory compression, using `-r`, thanks to Przemyslaw Skibinski +New : Command `--rm`, to remove source file after successful de/compression +New : Visual build scripts, by Christophe Chevalier +New : Support for Sparse File-systems (do not use space for zero-filled sectors) +New : Frame checksum support +New : Support pass-through mode (when using `-df`) +API : more efficient Dictionary API : `ZSTD_compress_usingCDict()`, `ZSTD_decompress_usingDDict()` +API : create dictionary files from custom content, by Giuseppe Ottaviano +API : support for custom malloc/free functions +New : controllable Dictionary ID +New : Support for skippable frames + +v0.6.1 (May 13, 2016) +New : zlib wrapper API, thanks to Przemyslaw Skibinski +New : Ability to compile compressor / decompressor separately +Changed : new lib directory structure +Fixed : Legacy codec v0.5 compatible with dictionary decompression +Fixed : Decoder corruption error (#173) +Fixed : null-string roundtrip (#176) +New : benchmark mode can select directory as input +Experimental : midipix support, VMS support + +v0.6.0 (Apr 13, 2016) +Stronger high compression modes, thanks to Przemyslaw Skibinski +API : ZSTD_getFrameParams() provides size of decompressed content +New : highest compression modes require `--ultra` command to fully unleash their capacity +Fixed : zstd cli return error code > 0 and removes dst file artifact when decompression fails, thanks to Chip Turner + +v0.5.1 (Feb 18, 2016) +New : Optimal parsing => Very high compression modes, thanks to Przemyslaw Skibinski +Changed : Dictionary builder integrated into libzstd and zstd cli +Changed (!) : zstd cli now uses "multiple input files" as default mode. See `zstd -h`. +Fix : high compression modes for big-endian platforms +New : zstd cli : `-t` | `--test` command + +v0.5.0 (Feb 5, 2016) +New : dictionary builder utility +Changed : streaming & dictionary API +Improved : better compression of small data + +v0.4.7 (Jan 22, 2016) +Improved : small compression speed improvement in HC mode +Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default +fix : bt search bug + +v0.4.6 (Jan 13, 2016) +fix : fast compression mode on Windows +New : cmake configuration file, thanks to Artyom Dymchenko +Improved : high compression mode on repetitive data +New : block-level API +New : ZSTD_duplicateCCtx() + +v0.4.5 (Dec 18, 2015) +new : -m/--multiple : compress/decompress multiple files + +v0.4.4 (Dec 14, 2015) +Fixed : high compression modes for Windows 32 bits +new : external dictionary API extended to buffered mode and accessible through command line +new : windows DLL project, thanks to Christophe Chevalier + +v0.4.3 (Dec 7, 2015) +new : external dictionary API +new : zstd-frugal + +v0.4.2 (Dec 2, 2015) +Generic minor improvements for small blocks +Fixed : big-endian compatibility, by Peter Harris (#85) + +v0.4.1 (Dec 1, 2015) +Fixed : ZSTD_LEGACY_SUPPORT=0 build mode (reported by Luben) +removed `zstd.c` + +v0.4.0 (Nov 29, 2015) +Command line utility compatible with high compression levels +Removed zstdhc => merged into zstd +Added : ZBUFF API (see zstd_buffered.h) +Rolling buffer support + +v0.3.6 (Nov 10, 2015) +small blocks params + +v0.3.5 (Nov 9, 2015) +minor generic compression improvements + +v0.3.4 (Nov 6, 2015) +Faster fast cLevels + +v0.3.3 (Nov 5, 2015) +Small compression ratio improvement + +v0.3.2 (Nov 2, 2015) +Fixed Visual Studio + +v0.3.1 (Nov 2, 2015) +Small compression ratio improvement + +v0.3 (Oct 30, 2015) +HC mode : compression levels 2-26 + +v0.2.2 (Oct 28, 2015) +Fix : Visual Studio 2013 & 2015 release compilation, by Christophe Chevalier + +v0.2.1 (Oct 24, 2015) +Fix : Read errors, advanced fuzzer tests, by Hanno Böck + +v0.2.0 (Oct 22, 2015) +**Breaking format change** +Faster decompression speed +Can still decode v0.1 format + +v0.1.3 (Oct 15, 2015) +fix uninitialization warning, reported by Evan Nemerson + +v0.1.2 (Sep 11, 2015) +frame concatenation support + +v0.1.1 (Aug 27, 2015) +fix compression bug +detects write-flush errors + +v0.1.0 (Aug 25, 2015) +first release diff --git a/External/Zstd/zstd-1.5.5/COPYING b/External/Zstd/zstd-1.5.5/COPYING new file mode 100644 index 000000000..ecbc05937 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. \ No newline at end of file diff --git a/External/Zstd/zstd-1.5.5/LICENSE b/External/Zstd/zstd-1.5.5/LICENSE new file mode 100644 index 000000000..75800288c --- /dev/null +++ b/External/Zstd/zstd-1.5.5/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/External/Zstd/zstd-1.5.5/README.md b/External/Zstd/zstd-1.5.5/README.md new file mode 100644 index 000000000..f91e68fdb --- /dev/null +++ b/External/Zstd/zstd-1.5.5/README.md @@ -0,0 +1,223 @@ +

Zstandard

+ +__Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm, +targeting real-time compression scenarios at zlib-level and better compression ratios. +It's backed by a very fast entropy stage, provided by [Huff0 and FSE library](https://github.com/Cyan4973/FiniteStateEntropy). + +Zstandard's format is stable and documented in [RFC8878](https://datatracker.ietf.org/doc/html/rfc8878). Multiple independent implementations are already available. +This repository represents the reference implementation, provided as an open-source dual [BSD](LICENSE) and [GPLv2](COPYING) licensed **C** library, +and a command line utility producing and decoding `.zst`, `.gz`, `.xz` and `.lz4` files. +Should your project require another programming language, +a list of known ports and bindings is provided on [Zstandard homepage](https://facebook.github.io/zstd/#other-languages). + +**Development branch status:** + +[![Build Status][travisDevBadge]][travisLink] +[![Build status][CircleDevBadge]][CircleLink] +[![Build status][CirrusDevBadge]][CirrusLink] +[![Fuzzing Status][OSSFuzzBadge]][OSSFuzzLink] + +[travisDevBadge]: https://api.travis-ci.com/facebook/zstd.svg?branch=dev "Continuous Integration test suite" +[travisLink]: https://travis-ci.com/facebook/zstd +[CircleDevBadge]: https://circleci.com/gh/facebook/zstd/tree/dev.svg?style=shield "Short test suite" +[CircleLink]: https://circleci.com/gh/facebook/zstd +[CirrusDevBadge]: https://api.cirrus-ci.com/github/facebook/zstd.svg?branch=dev +[CirrusLink]: https://cirrus-ci.com/github/facebook/zstd +[OSSFuzzBadge]: https://oss-fuzz-build-logs.storage.googleapis.com/badges/zstd.svg +[OSSFuzzLink]: https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:zstd + +## Benchmarks + +For reference, several fast compression algorithms were tested and compared +on a desktop running Ubuntu 20.04 (`Linux 5.11.0-41-generic`), +with a Core i7-9700K CPU @ 4.9GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with [gcc] 9.3.0, +on the [Silesia compression corpus]. + +[lzbench]: https://github.com/inikep/lzbench +[Silesia compression corpus]: https://sun.aei.polsl.pl//~sdeor/index.php?page=silesia +[gcc]: https://gcc.gnu.org/ + +| Compressor name | Ratio | Compression| Decompress.| +| --------------- | ------| -----------| ---------- | +| **zstd 1.5.1 -1** | 2.887 | 530 MB/s | 1700 MB/s | +| [zlib] 1.2.11 -1 | 2.743 | 95 MB/s | 400 MB/s | +| brotli 1.0.9 -0 | 2.702 | 395 MB/s | 450 MB/s | +| **zstd 1.5.1 --fast=1** | 2.437 | 600 MB/s | 2150 MB/s | +| **zstd 1.5.1 --fast=3** | 2.239 | 670 MB/s | 2250 MB/s | +| quicklz 1.5.0 -1 | 2.238 | 540 MB/s | 760 MB/s | +| **zstd 1.5.1 --fast=4** | 2.148 | 710 MB/s | 2300 MB/s | +| lzo1x 2.10 -1 | 2.106 | 660 MB/s | 845 MB/s | +| [lz4] 1.9.3 | 2.101 | 740 MB/s | 4500 MB/s | +| lzf 3.6 -1 | 2.077 | 410 MB/s | 830 MB/s | +| snappy 1.1.9 | 2.073 | 550 MB/s | 1750 MB/s | + +[zlib]: https://www.zlib.net/ +[lz4]: https://lz4.github.io/lz4/ + +The negative compression levels, specified with `--fast=#`, +offer faster compression and decompression speed +at the cost of compression ratio (compared to level 1). + +Zstd can also offer stronger compression ratios at the cost of compression speed. +Speed vs Compression trade-off is configurable by small increments. +Decompression speed is preserved and remains roughly the same at all settings, +a property shared by most LZ compression algorithms, such as [zlib] or lzma. + +The following tests were run +on a server running Linux Debian (`Linux version 4.14.0-3-amd64`) +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with [gcc] 7.3.0, +on the [Silesia compression corpus]. + +Compression Speed vs Ratio | Decompression Speed +---------------------------|-------------------- +![Compression Speed vs Ratio](doc/images/CSpeed2.png "Compression Speed vs Ratio") | ![Decompression Speed](doc/images/DSpeed3.png "Decompression Speed") + +A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph. +For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png). + + +## The case for Small Data compression + +Previous charts provide results applicable to typical file and stream scenarios (several MB). Small data comes with different perspectives. + +The smaller the amount of data to compress, the more difficult it is to compress. This problem is common to all compression algorithms, and reason is, compression algorithms learn from past data how to compress future data. But at the beginning of a new data set, there is no "past" to build upon. + +To solve this situation, Zstd offers a __training mode__, which can be used to tune the algorithm for a selected type of data. +Training Zstandard is achieved by providing it with a few samples (one file per sample). The result of this training is stored in a file called "dictionary", which must be loaded before compression and decompression. +Using this dictionary, the compression ratio achievable on small data improves dramatically. + +The following example uses the `github-users` [sample set](https://github.com/facebook/zstd/releases/tag/v1.1.3), created from [github public API](https://developer.github.com/v3/users/#get-all-users). +It consists of roughly 10K records weighing about 1KB each. + +Compression Ratio | Compression Speed | Decompression Speed +------------------|-------------------|-------------------- +![Compression Ratio](doc/images/dict-cr.png "Compression Ratio") | ![Compression Speed](doc/images/dict-cs.png "Compression Speed") | ![Decompression Speed](doc/images/dict-ds.png "Decompression Speed") + + +These compression gains are achieved while simultaneously providing _faster_ compression and decompression speeds. + +Training works if there is some correlation in a family of small data samples. The more data-specific a dictionary is, the more efficient it is (there is no _universal dictionary_). +Hence, deploying one dictionary per type of data will provide the greatest benefits. +Dictionary gains are mostly effective in the first few KB. Then, the compression algorithm will gradually use previously decoded content to better compress the rest of the file. + +### Dictionary compression How To: + +1. Create the dictionary + + `zstd --train FullPathToTrainingSet/* -o dictionaryName` + +2. Compress with dictionary + + `zstd -D dictionaryName FILE` + +3. Decompress with dictionary + + `zstd -D dictionaryName --decompress FILE.zst` + + +## Build instructions + +`make` is the officially maintained build system of this project. +All other build systems are "compatible" and 3rd-party maintained, +they may feature small differences in advanced options. +When your system allows it, prefer using `make` to build `zstd` and `libzstd`. + +### Makefile + +If your system is compatible with standard `make` (or `gmake`), +invoking `make` in root directory will generate `zstd` cli in root directory. +It will also create `libzstd` into `lib/`. + +Other available options include: +- `make install` : create and install zstd cli, library and man pages +- `make check` : create and run `zstd`, test its behavior on local platform + +The `Makefile` follows the [GNU Standard Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html), +allowing staged install, standard flags, directory variables and command variables. + +For advanced use cases, specialized compilation flags which control binary generation +are documented in [`lib/README.md`](lib/README.md#modular-build) for the `libzstd` library +and in [`programs/README.md`](programs/README.md#compilation-variables) for the `zstd` CLI. + +### cmake + +A `cmake` project generator is provided within `build/cmake`. +It can generate Makefiles or other build scripts +to create `zstd` binary, and `libzstd` dynamic and static libraries. + +By default, `CMAKE_BUILD_TYPE` is set to `Release`. + +#### Support for Fat (Universal2) Output + +`zstd` can be built and installed with support for both Apple Silicon (M1/M2) as well as Intel by using CMake's Universal2 support. +To perform a Fat/Universal2 build and install use the following commands: + +```bash +cmake -B build-cmake-debug -S build/cmake -G Ninja -DCMAKE_OSX_ARCHITECTURES="x86_64;x86_64h;arm64" +cd build-cmake-debug +ninja +sudo ninja install +``` + +### Meson + +A Meson project is provided within [`build/meson`](build/meson). Follow +build instructions in that directory. + +You can also take a look at [`.travis.yml`](.travis.yml) file for an +example about how Meson is used to build this project. + +Note that default build type is **release**. + +### VCPKG +You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install zstd + +The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors. +If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. + +### Visual Studio (Windows) + +Going into `build` directory, you will find additional possibilities: +- Projects for Visual Studio 2005, 2008 and 2010. + + VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017. +- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`, + which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. + +### Buck + +You can build the zstd binary via buck by executing: `buck build programs:zstd` from the root of the repo. +The output binary will be in `buck-out/gen/programs/`. + +## Testing + +You can run quick local smoke tests by running `make check`. +If you can't use `make`, execute the `playTest.sh` script from the `src/tests` directory. +Two env variables `$ZSTD_BIN` and `$DATAGEN_BIN` are needed for the test script to locate the `zstd` and `datagen` binary. +For information on CI testing, please refer to `TESTING.md`. + +## Status + +Zstandard is currently deployed within Facebook and many other large cloud infrastructures. +It is run continuously to compress large amounts of data in multiple formats and use cases. +Zstandard is considered safe for production environments. + +## License + +Zstandard is dual-licensed under [BSD](LICENSE) and [GPLv2](COPYING). + +## Contributing + +The `dev` branch is the one where all contributions are merged before reaching `release`. +If you plan to propose a patch, please commit into the `dev` branch, or its own feature branch. +Direct commit to `release` are not permitted. +For more information, please read [CONTRIBUTING](CONTRIBUTING.md). diff --git a/External/Zstd/zstd-1.5.5/lib/README.md b/External/Zstd/zstd-1.5.5/lib/README.md new file mode 100644 index 000000000..c3b5d1817 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/README.md @@ -0,0 +1,224 @@ +Zstandard library files +================================ + +The __lib__ directory is split into several sub-directories, +in order to make it easier to select or exclude features. + + +#### Building + +`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions), +including commands variables, staged install, directory variables and standard targets. +- `make` : generates both static and dynamic libraries +- `make install` : install libraries and headers in target system directories + +`libzstd` default scope is pretty large, including compression, decompression, dictionary builder, +and support for decoding legacy formats >= v0.5.0. +The scope can be reduced on demand (see paragraph _modular build_). + + +#### Multithreading support + +When building with `make`, by default the dynamic library is multithreaded and static library is single-threaded (for compatibility reasons). + +Enabling multithreading requires 2 conditions : +- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`) +- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`) + +For convenience, we provide a build target to generate multi and single threaded libraries: +- Force enable multithreading on both dynamic and static libraries by appending `-mt` to the target, e.g. `make lib-mt`. +- Force disable multithreading on both dynamic and static libraries by appending `-nomt` to the target, e.g. `make lib-nomt`. +- By default, as mentioned before, dynamic library is multithreaded, and static library is single-threaded, e.g. `make lib`. + +When linking a POSIX program with a multithreaded version of `libzstd`, +note that it's necessary to invoke the `-pthread` flag during link stage. + +Multithreading capabilities are exposed +via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.4.3/lib/zstd.h#L351). + + +#### API + +Zstandard's stable API is exposed within [lib/zstd.h](zstd.h). + + +#### Advanced API + +Optional advanced features are exposed via : + +- `lib/zstd_errors.h` : translates `size_t` function results + into a `ZSTD_ErrorCode`, for accurate error handling. + +- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`, + it unlocks access to the experimental API, + exposed in the second part of `zstd.h`. + All definitions in the experimental APIs are unstable, + they may still change in the future, or even be removed. + As a consequence, experimental definitions shall ___never be used with dynamic library___ ! + Only static linking is allowed. + + +#### Modular build + +It's possible to compile only a limited set of features within `libzstd`. +The file structure is designed to make this selection manually achievable for any build system : + +- Directory `lib/common` is always required, for all variants. + +- Compression source code lies in `lib/compress` + +- Decompression source code lies in `lib/decompress` + +- It's possible to include only `compress` or only `decompress`, they don't depend on each other. + +- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples. + The API is exposed in `lib/dictBuilder/zdict.h`. + This module depends on both `lib/common` and `lib/compress` . + +- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`. + This module depends on `lib/common` and `lib/decompress`. + To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation. + Specifying a number limits versions supported to that version onward. + For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0". + Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats". + By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`. + Decoding supported legacy format is a transparent capability triggered within decompression functions. + It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`. + Each version does also provide its own set of advanced API. + For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` . + +- While invoking `make libzstd`, it's possible to define build macros + `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`, + and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the + corresponding features. This will also disable compilation of all + dependencies (e.g. `ZSTD_LIB_COMPRESSION=0` will also disable + dictBuilder). + +- There are a number of options that can help minimize the binary size of + `libzstd`. + + The first step is to select the components needed (using the above-described + `ZSTD_LIB_COMPRESSION` etc.). + + The next step is to set `ZSTD_LIB_MINIFY` to `1` when invoking `make`. This + disables various optional components and changes the compilation flags to + prioritize space-saving. + + Detailed options: Zstandard's code and build environment is set up by default + to optimize above all else for performance. In pursuit of this goal, Zstandard + makes significant trade-offs in code size. For example, Zstandard often has + more than one implementation of a particular component, with each + implementation optimized for different scenarios. For example, the Huffman + decoder has complementary implementations that decode the stream one symbol at + a time or two symbols at a time. Zstd normally includes both (and dispatches + between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1` or + `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding + compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` + and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of + only one or the other of two decompression implementations. The smallest + binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and + `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`). + + For squeezing the last ounce of size out, you can also define + `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`, + which removes the error messages that are otherwise returned by + `ZSTD_getErrorName` (implied by `ZSTD_LIB_MINIFY`). + + Finally, when integrating into your application, make sure you're doing link- + time optimization and unused symbol garbage collection (via some combination of, + e.g., `-flto`, `-ffat-lto-objects`, `-fuse-linker-plugin`, + `-ffunction-sections`, `-fdata-sections`, `-fmerge-all-constants`, + `-Wl,--gc-sections`, `-Wl,-z,norelro`, and an archiver that understands + the compiler's intermediate representation, e.g., `AR=gcc-ar`). Consult your + compiler's documentation. + +- While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1` + will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in + the shared library, which is now hidden by default. + +- The build macro `DYNAMIC_BMI2` can be set to 1 or 0 in order to generate binaries + which can detect at runtime the presence of BMI2 instructions, and use them only if present. + These instructions contribute to better performance, notably on the decoder side. + By default, this feature is automatically enabled on detecting + the right instruction set (x64) and compiler (clang or gcc >= 5). + It's obviously disabled for different cpus, + or when BMI2 instruction set is _required_ by the compiler command line + (in this case, only the BMI2 code path is generated). + Setting this macro will either force to generate the BMI2 dispatcher (1) + or prevent it (0). It overrides automatic detection. + +- The build macro `ZSTD_NO_UNUSED_FUNCTIONS` can be defined to hide the definitions of functions + that zstd does not use. Not all unused functions are hidden, but they can be if needed. + Currently, this macro will hide function definitions in FSE and HUF that use an excessive + amount of stack space. + +- The build macro `ZSTD_NO_INTRINSICS` can be defined to disable all explicit intrinsics. + Compiler builtins are still used. + +- The build macro `ZSTD_DECODER_INTERNAL_BUFFER` can be set to control + the amount of extra memory used during decompression to store literals. + This defaults to 64kB. Reducing this value reduces the memory footprint of + `ZSTD_DCtx` decompression contexts, + but might also result in a small decompression speed cost. + +- The C compiler macros `ZSTDLIB_VISIBLE`, `ZSTDERRORLIB_VISIBLE` and `ZDICTLIB_VISIBLE` + can be overridden to control the visibility of zstd's API. Additionally, + `ZSTDLIB_STATIC_API` and `ZDICTLIB_STATIC_API` can be overridden to control the visibility + of zstd's static API. Specifically, it can be set to `ZSTDLIB_HIDDEN` to hide the symbols + from the shared library. These macros default to `ZSTDLIB_VISIBILITY`, + `ZSTDERRORLIB_VSIBILITY`, and `ZDICTLIB_VISIBILITY` if unset, for backwards compatibility + with the old macro names. + +#### Windows : using MinGW+MSYS to create DLL + +DLL can be created using MinGW+MSYS with the `make libzstd` command. +This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`. +The import library is only required with Visual C++. +The header file `zstd.h` and the dynamic library `dll\libzstd.dll` are required to +compile a project using gcc/MinGW. +The dynamic library has to be added to linking options. +It means that if a project that uses ZSTD consists of a single `test-dll.c` +file it should be linked with `dll\libzstd.dll`. For example: +``` + gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll +``` +The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`. + + +#### Advanced Build options + +The build system requires a hash function in order to +separate object files created with different compilation flags. +By default, it tries to use `md5sum` or equivalent. +The hash function can be manually switched by setting the `HASH` variable. +For example : `make HASH=xxhsum` +The hash function needs to generate at least 64-bit using hexadecimal format. +When no hash function is found, +the Makefile just generates all object files into the same default directory, +irrespective of compilation flags. +This functionality only matters if `libzstd` is compiled multiple times +with different build flags. + +The build directory, where object files are stored +can also be manually controlled using variable `BUILD_DIR`, +for example `make BUILD_DIR=objectDir/v1`. +In which case, the hash function doesn't matter. + + +#### Deprecated API + +Obsolete API on their way out are stored in directory `lib/deprecated`. +At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`. +These prototypes will be removed in some future version. +Consider migrating code towards supported streaming API exposed in `zstd.h`. + + +#### Miscellaneous + +The other files are not source code. There are : + + - `BUCK` : support for `buck` build system (https://buckbuild.com/) + - `Makefile` : `make` script to build and install zstd library (static and dynamic) + - `README.md` : this file + - `dll/` : resources directory for Windows compilation + - `libzstd.pc.in` : script for `pkg-config` (used in `make install`) diff --git a/External/Zstd/zstd-1.5.5/lib/common/allocations.h b/External/Zstd/zstd-1.5.5/lib/common/allocations.h new file mode 100644 index 000000000..a3153c4ba --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/allocations.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides custom allocation primitives + */ + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + +#include "mem.h" /* MEM_STATIC */ +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" /* ZSTD_customMem */ + +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H + +/* custom memory allocation functions */ + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +#endif /* ZSTD_ALLOCATIONS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/bits.h b/External/Zstd/zstd-1.5.5/lib/common/bits.h new file mode 100644 index 000000000..def56c474 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/bits.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +#include "mem.h" + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +# else + return ZSTD_countTrailingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) { + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +# else + return ZSTD_countLeadingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/bitstream.h b/External/Zstd/zstd-1.5.5/lib/common/bitstream.h new file mode 100644 index 000000000..72b0b3df2 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/bitstream.h @@ -0,0 +1,437 @@ +/* ****************************************************************** + * bitstream + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "compiler.h" /* UNLIKELY() */ +#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ +#include "error_private.h" /* error codes and messages */ +#include "bits.h" /* ZSTD_highbit32 */ + + +/*========================================= +* Target specific +=========================================*/ +#ifndef ZSTD_NO_INTRINSICS +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ +# elif defined(__ICCARM__) +# include +# endif +#endif + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct { + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS) + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; + + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; + + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; + + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; + + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; + + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer)*8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X86) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); +#else + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +#endif +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +MEM_STATIC FORCE_INLINE_ATTR void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not read beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +MEM_STATIC FORCE_INLINE_ATTR BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->limitPtr) { + return BIT_reloadDStreamFast(bitD); + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/compiler.h b/External/Zstd/zstd-1.5.5/lib/common/compiler.h new file mode 100644 index 000000000..73f8d0199 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/compiler.h @@ -0,0 +1,358 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +#include "portability_macros.h" + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ + +#if !defined(ZSTD_NO_INLINE) +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + +/** + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explicitly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#if defined(_MSC_VER) +# define WIN_CDECL __cdecl +#else +# define WIN_CDECL +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to eliminate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR +#endif + +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + +/* force no inlining */ +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/* target attribute */ +#if defined(__GNUC__) || defined(__ICCARM__) +# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) +#else +# define TARGET_ATTRIBUTE(target) +#endif + +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. + */ +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if defined(NO_PREFETCH) +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))) +# define PREFETCH_L2(ptr) __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))) +# else +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ +} + +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif +#else +# define DONT_VECTORIZE +#endif + +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#if defined(__GNUC__) +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE { assert(0), __builtin_unreachable(); } +#else +# define ZSTD_UNREACHABLE { assert(0); } +#endif + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +# include /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ +#ifndef STATIC_BMI2 +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) +# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 +# define STATIC_BMI2 1 +# endif +# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__) +# define STATIC_BMI2 1 +# endif +#endif + +#ifndef STATIC_BMI2 + #define STATIC_BMI2 0 +#endif + +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include +# endif +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +/*-************************************************************** +* Alignment check +*****************************************************************/ + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ +#define ZSTD_DEPS_NEED_STDINT +#include "zstd_deps.h" /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/cpu.h b/External/Zstd/zstd-1.5.5/lib/common/cpu.h new file mode 100644 index 000000000..8bc34a36d --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/cpu.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include "mem.h" + +#ifdef _MSC_VER +#include +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/debug.c b/External/Zstd/zstd-1.5.5/lib/common/debug.c new file mode 100644 index 000000000..ebf7bfccf --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/debug.c @@ -0,0 +1,24 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * This module only hosts one global variable + * which can be used to dynamically influence the verbosity of traces, + * such as DEBUGLOG and RAWLOG + */ + +#include "debug.h" + +int g_debuglevel = DEBUGLEVEL; diff --git a/External/Zstd/zstd-1.5.5/lib/common/debug.h b/External/Zstd/zstd-1.5.5/lib/common/debug.h new file mode 100644 index 000000000..0e9817ea6 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/debug.h @@ -0,0 +1,107 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * The purpose of this header is to enable debug functions. + * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, + * and DEBUG_STATIC_ASSERT() for compile-time. + * + * By default, DEBUGLEVEL==0, which means run-time debug is disabled. + * + * Level 1 enables assert() only. + * Starting level 2, traces can be generated and pushed to stderr. + * The higher the level, the more verbose the traces. + * + * It's possible to dynamically adjust level using variable g_debug_level, + * which is only declared if DEBUGLEVEL>=2, + * and is a global variable, not multi-thread protected (use with care) + */ + +#ifndef DEBUG_H_12987983217 +#define DEBUG_H_12987983217 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + + +/* DEBUGLEVEL is expected to be defined externally, + * typically through compiler command line. + * Value must be a number. */ +#ifndef DEBUGLEVEL +# define DEBUGLEVEL 0 +#endif + + +/* recommended values for DEBUGLEVEL : + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display + * 2 : reserved, for currently active debug path + * 3 : events once per object lifetime (CCtx, CDict, etc.) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (verbose) + * 7+: events at every position (*very* verbose) + * + * It's generally inconvenient to output traces > 5. + * In which case, it's possible to selectively trigger high verbosity levels + * by modifying g_debug_level. + */ + +#if (DEBUGLEVEL>=1) +# define ZSTD_DEPS_NEED_ASSERT +# include "zstd_deps.h" +#else +# ifndef assert /* assert may be already defined, due to prior #include */ +# define assert(condition) ((void)0) /* disable assert (default) */ +# endif +#endif + +#if (DEBUGLEVEL>=2) +# define ZSTD_DEPS_NEED_IO +# include "zstd_deps.h" +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ + +# define RAWLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } } +# define DEBUGLOG(l, ...) { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } } +#else +# define RAWLOG(l, ...) {} /* disabled */ +# define DEBUGLOG(l, ...) {} /* disabled */ +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* DEBUG_H_12987983217 */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/entropy_common.c b/External/Zstd/zstd-1.5.5/lib/common/entropy_common.c new file mode 100644 index 000000000..e2173afb0 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/entropy_common.c @@ -0,0 +1,340 @@ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#include "huf.h" +#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */ + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; + int previous0 = 0; + + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); + { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, + buffer, sizeof(buffer)); + if (FSE_isError(countSize)) return countSize; + if (countSize > hbSize) return ERROR(corruption_detected); + return countSize; + } } + assert(hbSize >= 8); + + /* init */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; + } else { + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; + } + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; + bitCount += 2; + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = ZSTD_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); + } + if (charnum >= maxSV1) break; + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } } + if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int flags) +{ +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)flags; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/External/Zstd/zstd-1.5.5/lib/common/error_private.c b/External/Zstd/zstd-1.5.5/lib/common/error_private.c new file mode 100644 index 000000000..075fc5ef4 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/error_private.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Data corruption detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; + case PREFIX(maxCode): + default: return notErrorCode; + } +#endif +} diff --git a/External/Zstd/zstd-1.5.5/lib/common/error_private.h b/External/Zstd/zstd-1.5.5/lib/common/error_private.h new file mode 100644 index 000000000..325daad40 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/error_private.h @@ -0,0 +1,159 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include "../zstd_errors.h" /* enum list */ +#include "compiler.h" +#include "debug.h" +#include "zstd_deps.h" /* size_t */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* already defined on Visual Studio */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + +/* check and forward error code */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return e +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0); + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0); + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/fse.h b/External/Zstd/zstd-1.5.5/lib/common/fse.h new file mode 100644 index 000000000..02a1f0bc5 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/fse.h @@ -0,0 +1,639 @@ +/* ****************************************************************** + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. + */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) +size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ + +typedef enum { + FSE_repeat_none, /**< Cannot use the previous table */ + FSE_repeat_check, /**< Can use the previous table but it must be checked */ + FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } FSE_repeat; + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + Hence their body are included in next section. +*/ +typedef struct { + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/**< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct { + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/**< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits+1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ + { U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; + } +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/common/fse_decompress.c b/External/Zstd/zstd-1.5.5/lib/common/fse_decompress.c new file mode 100644 index 000000000..1e1c9f92d --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/fse_decompress.c @@ -0,0 +1,311 @@ +/* ****************************************************************** + * FSE : Finite State Entropy decoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "debug.h" /* assert */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" +#include "bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; + FSE_DTable dtable[1]; /* Dynamically sized */ +} FSE_DecompressWksp; + + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + + DEBUG_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* normal FSE decoding mode */ + { + size_t const NCountLength = FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } + + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = wksp->dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, wksp->dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); +} + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/huf.h b/External/Zstd/zstd-1.5.5/lib/common/huf.h new file mode 100644 index 000000000..73d1ee565 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/huf.h @@ -0,0 +1,273 @@ +/* ****************************************************************** + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include "zstd_deps.h" /* size_t */ +#include "mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ + +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; + + +/* **************************************** + * HUF detailed API + * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } HUF_repeat; + +/** HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_buildCTable_wksp (HUF_CElt* tree, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int flags); + +/** HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); + +/** HUF_getNbBitsFromCTable() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : is not inlined, as HUF_CElt definition is private */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +/** HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#endif +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif + +#endif /* HUF_H_298734234 */ + +#if defined (__cplusplus) +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/common/mem.h b/External/Zstd/zstd-1.5.5/lib/common/mem.h new file mode 100644 index 000000000..98dd47a04 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/mem.h @@ -0,0 +1,435 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include "compiler.h" /* __has_builtin */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "zstd_deps.h" /* ZSTD_memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if defined(_AIX) +# include +# else +# include /* intptr_t */ +# endif + typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else +# include +#if CHAR_BIT != 8 +# error "this implementation requires char to be exactly 8-bit type" +#endif + typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; +#if USHRT_MAX != 65535 +# error "this implementation requires short to be exactly 16-bit type" +#endif + typedef unsigned short U16; + typedef signed short S16; +#if UINT_MAX != 4294967295 +# error "this implementation requires int to be exactly 32-bit type" +#endif + typedef unsigned int U32; + typedef signed int S32; +/* note : there are no limits defined for long long type in C90. + * limits exist in C99, however, in such case, is preferred */ + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * Default : method 1 if supported, else method 0 + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# ifdef __GNUC__ +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#else + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +#endif +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return MEM_swap32_fallback(in); +#endif +} + +MEM_STATIC U64 MEM_swap64_fallback(U64 in) +{ + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return MEM_swap64_fallback(in); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +/* code only tested on 32 and 64 bits systems */ +MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/pool.c b/External/Zstd/zstd-1.5.5/lib/common/pool.c new file mode 100644 index 000000000..d5ca5a780 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/pool.c @@ -0,0 +1,371 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Dependencies ======= */ +#include "../common/allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */ +#include "zstd_deps.h" /* size_t */ +#include "debug.h" /* assert */ +#include "pool.h" + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +#ifdef ZSTD_MULTITHREAD + +#include "threading.h" /* pthread adaptation */ + +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void *opaque; +} POOL_job; + +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t* threads; + size_t threadCapacity; + size_t threadLimit; + + /* The queue is a circular buffer */ + POOL_job *queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; + + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; + +/* POOL_thread() : + * Work thread for the thread pool. + * Waits for jobs and executes them. + * @returns : NULL on failure else non-null. + */ +static void* POOL_thread(void* opaque) { + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { return NULL; } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + + while ( ctx->queueEmpty + || (ctx->numThreadsBusy >= ctx->threadLimit) ) { + if (ctx->shutdown) { + /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), + * a few threads will be shutdown while !queueEmpty, + * but enough threads will remain active to finish the queue */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* Pop a job off the queue */ + { POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = (ctx->queueHead == ctx->queueTail); + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + + job.function(job.opaque); + + /* If the intended queue size was 0, signal after finishing job */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + } + } /* for (;;) */ + assert(0); /* Unreachable */ +} + +/* ZSTD_createThreadPool() : public access point */ +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem) +{ + POOL_ctx* ctx; + /* Check parameters */ + if (!numThreads) { return NULL; } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); + if (!ctx) { return NULL; } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate + * empty and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threadCapacity = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } + /* Initialize the threads */ + { size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = i; + POOL_free(ctx); + return NULL; + } } + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + } + return ctx; +} + +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) { + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { size_t i; + for (i = 0; i < ctx->threadCapacity; ++i) { + ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */ + } } +} + +void POOL_free(POOL_ctx *ctx) { + if (!ctx) { return; } + POOL_join(ctx); + ZSTD_pthread_mutex_destroy(&ctx->queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_customFree(ctx->queue, ctx->customMem); + ZSTD_customFree(ctx->threads, ctx->customMem); + ZSTD_customFree(ctx, ctx->customMem); +} + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +} + + +/* @return : 0 on success, 1 on error */ +static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +{ + if (numThreads <= ctx->threadCapacity) { + if (!numThreads) return 1; + ctx->threadLimit = numThreads; + return 0; + } + /* numThreads > threadCapacity */ + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + if (!threadPool) return 1; + /* replace existing thread pool */ + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); + ZSTD_customFree(ctx->threads, ctx->customMem); + ctx->threads = threadPool; + /* Initialize additional threads */ + { size_t threadId; + for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { + if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = threadId; + return 1; + } } + } } + /* successfully expanded */ + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + return 0; +} + +/* @return : 0 on success, 1 on error */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + int result; + if (ctx==NULL) return 1; + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + result = POOL_resize_internal(ctx, numThreads); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return result; +} + +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * When queueSize is 1 (pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free _and_ no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) { + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return (ctx->numThreadsBusy == ctx->threadLimit) || + !ctx->queueEmpty; + } +} + + +static void +POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) +{ + POOL_job job; + job.function = function; + job.opaque = opaque; + assert(ctx != NULL); + if (ctx->shutdown) return; + + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && (!ctx->shutdown)) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + if (isQueueFull(ctx)) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 0; + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 1; +} + + +#else /* ZSTD_MULTITHREAD not defined */ + +/* ========================== */ +/* No multi-threading support */ +/* ========================== */ + + +/* We don't need any data, but if it is empty, malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_poolCtx; + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* +POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_poolCtx; +} + +void POOL_free(POOL_ctx* ctx) { + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +int POOL_resize(POOL_ctx* ctx, size_t numThreads) { + (void)ctx; (void)numThreads; + return 0; +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); +} + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); + return 1; +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_poolCtx); + return sizeof(*ctx); +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/pool.h b/External/Zstd/zstd-1.5.5/lib/common/pool.h new file mode 100644 index 000000000..eb22ff509 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/pool.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef POOL_H +#define POOL_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +#include "zstd_deps.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ +#include "../zstd.h" + +typedef struct POOL_ctx_s POOL_ctx; + +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. +*/ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem); + +/*! POOL_free() : + * Free a thread pool returned by POOL_create(). + */ +void POOL_free(POOL_ctx* ctx); + + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx); + +/*! POOL_resize() : + * Expands or shrinks pool's number of threads. + * This is more efficient than releasing + creating a new context, + * since it tries to preserve and re-use existing threads. + * `numThreads` must be at least 1. + * @return : 0 when resize was successful, + * !0 (typically 1) if there is an error. + * note : only numThreads can be resized, queueSize remains unchanged. + */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads); + +/*! POOL_sizeof() : + * @return threadpool memory usage + * note : compatible with NULL (returns 0 in this case) + */ +size_t POOL_sizeof(const POOL_ctx* ctx); + +/*! POOL_function : + * The function type that can be added to a thread pool. + */ +typedef void (*POOL_function)(void*); + +/*! POOL_add() : + * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. + * Possibly blocks until there is room in the queue. + * Note : The function may be executed asynchronously, + * therefore, `opaque` must live until function has been completed. + */ +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); + + +/*! POOL_tryAdd() : + * Add the job `function(opaque)` to thread pool _if_ a queue slot is available. + * Returns immediately even if not (does not block). + * @return : 1 if successful, 0 if not. + */ +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); + + +#if defined (__cplusplus) +} +#endif + +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/common/portability_macros.h b/External/Zstd/zstd-1.5.5/lib/common/portability_macros.h new file mode 100644 index 000000000..8fd6ea82d --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/portability_macros.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X64)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/** + * Only enable assembly for GNUC compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/threading.c b/External/Zstd/zstd-1.5.5/lib/common/threading.c new file mode 100644 index 000000000..ca155b9b9 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/threading.c @@ -0,0 +1,176 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This file will hold wrapper for systems, which do not support pthreads + */ + +#include "threading.h" + +/* create fake symbol to avoid empty translation unit warning */ +int g_ZSTD_threading_useless_symbol; + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ + + +/* === Dependencies === */ +#include +#include + + +/* === Implementation === */ + +typedef struct { + void* (*start_routine)(void*); + void* arg; + int initialized; + ZSTD_pthread_cond_t initialized_cond; + ZSTD_pthread_mutex_t initialized_mutex; +} ZSTD_thread_params_t; + +static unsigned __stdcall worker(void *arg) +{ + void* (*start_routine)(void*); + void* thread_arg; + + /* Initialized thread_arg and start_routine and signal main thread that we don't need it + * to wait any longer. + */ + { + ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg; + thread_arg = thread_param->arg; + start_routine = thread_param->start_routine; + + /* Signal main thread that we are running and do not depend on its memory anymore */ + ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex); + thread_param->initialized = 1; + ZSTD_pthread_cond_signal(&thread_param->initialized_cond); + ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex); + } + + start_routine(thread_arg); + + return 0; +} + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg) +{ + ZSTD_thread_params_t thread_param; + (void)unused; + + thread_param.start_routine = start_routine; + thread_param.arg = arg; + thread_param.initialized = 0; + *thread = NULL; + + /* Setup thread initialization synchronization */ + if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) { + /* Should never happen on Windows */ + return -1; + } + if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) { + /* Should never happen on Windows */ + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return -1; + } + + /* Spawn thread */ + *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL); + if (!thread) { + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return errno; + } + + /* Wait for thread to be initialized */ + ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex); + while(!thread_param.initialized) { + ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex); + } + ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex); + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + + return 0; +} + +int ZSTD_pthread_join(ZSTD_pthread_t thread) +{ + DWORD result; + + if (!thread) return 0; + + result = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + + switch (result) { + case WAIT_OBJECT_0: + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif /* ZSTD_MULTITHREAD */ + +#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) +{ + *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); + if (!*mutex) + return 1; + return pthread_mutex_init(*mutex, attr); +} + +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) +{ + if (!*mutex) + return 0; + { + int const ret = pthread_mutex_destroy(*mutex); + ZSTD_free(*mutex); + return ret; + } +} + +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) +{ + *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); + if (!*cond) + return 1; + return pthread_cond_init(*cond, attr); +} + +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) +{ + if (!*cond) + return 0; + { + int const ret = pthread_cond_destroy(*cond); + ZSTD_free(*cond); + return ret; + } +} + +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/common/threading.h b/External/Zstd/zstd-1.5.5/lib/common/threading.h new file mode 100644 index 000000000..fb5c1c878 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/threading.h @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef THREADING_H_938743 +#define THREADING_H_938743 + +#include "debug.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ +#ifdef WINVER +# undef WINVER +#endif +#define WINVER 0x0600 + +#ifdef _WIN32_WINNT +# undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#include +#undef ERROR +#define ERROR(name) ZSTD_ERROR(name) + + +/* mutex */ +#define ZSTD_pthread_mutex_t CRITICAL_SECTION +#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0) +#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) +#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) + +/* condition variable */ +#define ZSTD_pthread_cond_t CONDITION_VARIABLE +#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) +#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) +#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) + +/* ZSTD_pthread_create() and ZSTD_pthread_join() */ +typedef HANDLE ZSTD_pthread_t; + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg); + +int ZSTD_pthread_join(ZSTD_pthread_t thread); + +/** + * add here more wrappers as required + */ + + +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +/* === POSIX Systems === */ +# include + +#if DEBUGLEVEL < 1 + +#define ZSTD_pthread_mutex_t pthread_mutex_t +#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) +#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) + +#define ZSTD_pthread_cond_t pthread_cond_t +#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) +#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#else /* DEBUGLEVEL >= 1 */ + +/* Debug implementation of threading. + * In this implementation we use pointers for mutexes and condition variables. + * This way, if we forget to init/destroy them the program will crash or ASAN + * will report leaks. + */ + +#define ZSTD_pthread_mutex_t pthread_mutex_t* +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr); +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex); +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a)) + +#define ZSTD_pthread_cond_t pthread_cond_t* +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr); +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#endif + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multithreading support */ + +typedef int ZSTD_pthread_mutex_t; +#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_mutex_destroy(a) ((void)(a)) +#define ZSTD_pthread_mutex_lock(a) ((void)(a)) +#define ZSTD_pthread_mutex_unlock(a) ((void)(a)) + +typedef int ZSTD_pthread_cond_t; +#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b)) +#define ZSTD_pthread_cond_signal(a) ((void)(a)) +#define ZSTD_pthread_cond_broadcast(a) ((void)(a)) + +/* do not use ZSTD_pthread_t */ + +#endif /* ZSTD_MULTITHREAD */ + +#if defined (__cplusplus) +} +#endif + +#endif /* THREADING_H_938743 */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/xxhash.c b/External/Zstd/zstd-1.5.5/lib/common/xxhash.c new file mode 100644 index 000000000..fd237c906 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/xxhash.c @@ -0,0 +1,24 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - xxHash homepage: https://cyan4973.github.io/xxHash/ + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/External/Zstd/zstd-1.5.5/lib/common/xxhash.h b/External/Zstd/zstd-1.5.5/lib/common/xxhash.h new file mode 100644 index 000000000..b8b73290b --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/xxhash.h @@ -0,0 +1,5686 @@ +/* + * xxHash - Fast Hash algorithm + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - xxHash homepage: https://cyan4973.github.io/xxHash/ + * - xxHash source repository : https://github.com/Cyan4973/xxHash + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +*/ + + +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif + +/*! + * @mainpage xxHash + * + * @file xxhash.h + * xxHash prototypes and implementation + */ +/* TODO: update */ +/* Notice extracted from xxHash homepage: + +xxHash is an extremely fast hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MurmurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +Note: SMHasher's CRC32 implementation is not the fastest one. +Other speed-oriented implementations can be faster, +especially in combination with PCLMUL instruction: +https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html?showComment=1552696407071#c3490092340461170735 + +A 64-bit version, named XXH64, is available since r35. +It offers much better speed, but for 64-bit applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * XXH_INLINE_ALL (and XXH_PRIVATE_API) + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + + + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + + +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 1 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return `XXH_VERSION_NUMBER` of the invoked library. + */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# else +# if ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +# endif +#endif + +/*! + * @} + * + * @defgroup xxh32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that @ref xxh3_family provides competitive speed + * for both 32-bit and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref xxh64_family, @ref xxh3_family : Other xxHash families + * @see @ref xxh32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit hash value. + * + * @see + * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +/*! + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * Example code for incrementally hashing a file: + * @code{.c} + * #include + * #include + * #define BUFFER_SIZE 256 + * + * // Note: XXH64 and XXH3 use the same interface. + * XXH32_hash_t + * hashFile(FILE* stream) + * { + * XXH32_state_t* state; + * unsigned char buf[BUFFER_SIZE]; + * size_t amt; + * XXH32_hash_t hash; + * + * state = XXH32_createState(); // Create a state + * assert(state != NULL); // Error check here + * XXH32_reset(state, 0xbaad5eed); // Reset state with our seed + * while ((amt = fread(buf, 1, sizeof(buf), stream)) != 0) { + * XXH32_update(state, buf, amt); // Hash the file in chunks + * } + * hash = XXH32_digest(state); // Finalize the hash + * XXH32_freeState(state); // Clean up + * return hash; + * } + * @endcode + */ + +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * Must be freed with XXH32_freeState(). + * @return An allocated XXH32_state_t on success, `NULL` on failure. + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * Must be allocated with XXH32_createState(). + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * @return XXH_OK. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * This function resets and seeds a state. Call it before @ref XXH32_update(). + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success, @ref XXH_ERROR on failure. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated xxHash32 value from that state. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +/******* Canonical representation *******/ + +/* + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + */ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif + +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* +Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute +introduced in CPP17 and C23. +CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough +C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough +*/ +#if XXH_HAS_C_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_CPP_ATTRIBUTE(x) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((fallthrough)) +#else +# define XXH_FALLTHROUGH +#endif + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup xxh64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * This function usually runs faster on 64-bit systems, but slower on 32-bit + * systems (see benchmark). + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit hash. + * + * @see + * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128(): + * Direct equivalents for the other variants of xxHash. + * @see + * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version. + */ +XXH_PUBLIC_API XXH64_hash_t XXH64(const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 +/*! + * @} + * ************************************************************************ + * @defgroup xxh3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Any 32-bit and 64-bit targets that can run XXH32 smoothly + * can run XXH3 at competitive speeds, even without vector support. + * Further details are explained in the implementation. + * + * Optimized implementations are provided for AVX512, AVX2, SSE2, NEON, POWER8, + * ZVector and scalar targets. This can be controlled via the XXH_VECTOR macro. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generage exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ + +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/* XXH3_64bits(): + * default 64-bit variant, using default secret and default seed of 0. + * It's the fastest variant. */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* data, size_t len); + +/* + * XXH3_64bits_withSeed(): + * This variant generates a custom secret on the fly + * based on default secret altered using the `seed` value. + * While this operation is decently fast, note that it's not completely free. + * Note: seed==0 produces the same results as XXH3_64bits(). + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/* + * XXH3_64bits_withSecret(): + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing "XXH3_generateSecret()" instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); +XXH_PUBLIC_API void XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state); + +/* + * XXH3_64bits_reset(): + * Initialize with default parameters. + * digest will be equivalent to `XXH3_64bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH3_state_t* statePtr); +/* + * XXH3_64bits_reset_withSeed(): + * Generate a custom secret from `seed`, and store it into `statePtr`. + * digest will be equivalent to `XXH3_64bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +/* + * XXH3_64bits_reset_withSecret(): + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* statePtr); + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* data, size_t len); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSeed(const void* data, size_t len, XXH64_hash_t seed); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_withSecret(const void* data, size_t len, const void* secret, size_t secretSize); + +/******* Streaming *******/ +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH3_state_t* statePtr); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed); +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize); + +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH3_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* statePtr); + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * XXH128_isEqual(): + * Return: 1 if `h1` and `h2` are equal, 0 if they are not. + */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * XXH128_cmp(): + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * return: >0 if *h128_1 > *h128_2 + * =0 if *h128_1 == *h128_2 + * <0 if *h128_1 < *h128_2 + */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash); +XXH_PUBLIC_API XXH128_hash_t XXH128_hashFromCanonical(const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. Similar to `vN` in @ref XXH32_state_s::v1 and @ref XXH64_state_s */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) { (XXH3_state_ptr)->seed = 0; } + + +/* XXH128() : + * simple alias to pre-selected XXH3_128bits variant + */ +XXH_PUBLIC_API XXH128_hash_t XXH128(const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/* + * XXH3_generateSecret(): + * + * Derive a high-entropy secret from any user-defined content, named customSeed. + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection than 64-bit seed, + * as it becomes much more difficult for an external actor to guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @secretSize + * into an already allocated buffer @secretBuffer. + * @secretSize must be >= XXH3_SECRET_SIZE_MIN + * + * The generated secret can then be used with any `*_withSecret()` variant. + * Functions `XXH3_128bits_withSecret()`, `XXH3_64bits_withSecret()`, + * `XXH3_128bits_reset_withSecret()` and `XXH3_64bits_reset_withSecret()` + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so + * XXH3_generateSecret() can be employed to ensure proper quality. + * + * customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch of zeroes. + * The resulting `secret` will nonetheless provide all required qualities. + * + * When customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize); + + +/* + * XXH3_generateSecret_fromSeed(): + * + * Generate the same secret as the _withSeed() variants. + * + * The resulting secret has a length of XXH3_SECRET_DEFAULT_SIZE (necessarily). + * @secretBuffer must be already allocated, of size at least XXH3_SECRET_DEFAULT_SIZE bytes. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * This generator is notably useful in combination with `_withSecretandSeed()`, + * as a way to emulate a faster `_withSeed()` variant. + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(void* secretBuffer, XXH64_hash_t seed); + +/* + * *_withSecretandSeed() : + * These variants generate hash values using either + * @seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes) + * or @secret for "large" keys (>= XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, + * via its impact to the seed. + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed); + +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* data, size_t len, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, + const void* secret, size_t secretSize, + XXH64_hash_t seed64); + + +#endif /* XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref xxh32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((packed))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * . + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64 & arm64, + * which are platforms known to offer good unaligned memory accesses performance. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), optimizing for size (-Os, -Oz), or using + * -fno-inline with GCC or Clang, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for gcc on armv7+ and mips */ +# if !defined(__clang__) && \ +( \ + (defined(__INTEL_COMPILER) && !defined(_WIN32)) || \ + ( \ + defined(__GNUC__) && ( \ + (defined(__ARM_ARCH) && __ARM_ARCH >= 7) || \ + ( \ + defined(__mips__) && \ + (__mips <= 5 || __mips_isa_rev < 6) && \ + (!defined(__mips16) || defined(__mips_mips16e2)) \ + ) \ + ) \ + ) \ +) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(__x86_64__) || defined(__aarch64__) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if defined(__OPTIMIZE_SIZE__) /* -Os, -Oz */ \ + || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for ZSTD_malloc(), ZSTD_free() */ +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* size_t, ZSTD_malloc, ZSTD_free, ZSTD_memcpy */ +static void* XXH_malloc(size_t s) { return ZSTD_malloc(s); } +static void XXH_free (void* p) { ZSTD_free(p); } +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_memcpy(dest,src,size); } + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# define XXH_ASSERT(c) ((void)0) +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# include +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (e.g. eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__ __volatile__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef union { xxh_u32 u32; } __attribute__((packed)) xxh_unalign; + return ((const xxh_unalign*)ptr)->u32; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup xxh32_impl XXH32 implementation + * @ingroup impl + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang autovectorizes it incorrectly + * and it is pointless writing a NEON implementation that is basically the + * same speed as scalar for XXH32. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param h32 The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 h32) +{ + h32 ^= h32 >> 15; + h32 *= XXH_PRIME32_2; + h32 ^= h32 >> 13; + h32 *= XXH_PRIME32_3; + h32 ^= h32 >> 16; + return(h32); +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param h32 The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + */ +static xxh_u32 +XXH32_finalize(xxh_u32 h32, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + h32 += (*ptr++) * XXH_PRIME32_5; \ + h32 = XXH_rotl32(h32, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + h32 += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + h32 = XXH_rotl32(h32, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(h32); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 4: XXH_PROCESS4; + return XXH32_avalanche(h32); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(h32); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; + case 0: return XXH32_avalanche(h32); + } + XXH_ASSERT(0); + return h32; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +/*! + * @ingroup xxh32_family + */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! + * @ingroup xxh32_family + * The default return values from XXH functions are unsigned 32 and 64 bit + * integers. + * + * The canonical representation uses big endian convention, the same convention + * as human-readable numbers (large digits first). + * + * This way, hash values can be written into a file or buffer, remaining + * comparable across different systems. + * + * The following functions allow transformation of hash values to and from their + * canonical format. + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + /* XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); */ + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup xxh32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __pack instructions are safer, but compiler specific, hence potentially + * problematic for some compilers. + * + * Currently only defined for GCC and ICC. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) xxh_unalign64; + return ((const xxh_unalign64*)ptr)->u64; +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup xxh64_impl XXH64 implementation + * @ingroup impl + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +static xxh_u64 XXH64_avalanche(xxh_u64 h64) +{ + h64 ^= h64 >> 33; + h64 *= XXH_PRIME64_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME64_3; + h64 ^= h64 >> 32; + return h64; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +static xxh_u64 +XXH64_finalize(xxh_u64 h64, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + h64 ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + h64 ^= (*ptr++) * XXH_PRIME64_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(h64); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (inputv[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH64_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} + + +/******* Canonical representation *******/ + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + /* XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); */ + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup xxh64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup xxh3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#else +/* Note: it might be useful to define __restrict or __restrict__ for some C++ compilers */ +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || defined(__aarch64__) || defined(_M_ARM) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * @ref XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< NEON for most ARMv7-A and all AArch64 */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && !defined(__OPTIMIZE_SIZE__) /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + + +#if XXH_VECTOR == XXH_NEON +/* + * NEON's setup for vmlal_u32 is a little more complicated than it is on + * SSE2, AVX2, and VSX. + * + * While PMULUDQ and VMULEUW both perform a mask, VMLAL.U32 performs an upcast. + * + * To do the same operation, the 128-bit 'Q' register needs to be split into + * two 64-bit 'D' registers, performing this operation:: + * + * [ a | b ] + * | '---------. .--------' | + * | x | + * | .---------' '--------. | + * [ a & 0xFFFFFFFF | b & 0xFFFFFFFF ],[ a >> 32 | b >> 32 ] + * + * Due to significant changes in aarch64, the fastest method for aarch64 is + * completely different than the fastest method for ARMv7-A. + * + * ARMv7-A treats D registers as unions overlaying Q registers, so modifying + * D11 will modify the high half of Q5. This is similar to how modifying AH + * will only affect bits 8-15 of AX on x86. + * + * VZIP takes two registers, and puts even lanes in one register and odd lanes + * in the other. + * + * On ARMv7-A, this strangely modifies both parameters in place instead of + * taking the usual 3-operand form. + * + * Therefore, if we want to do this, we can simply use a D-form VZIP.32 on the + * lower and upper halves of the Q register to end up with the high and low + * halves where we want - all in one instruction. + * + * vzip.32 d10, d11 @ d10 = { d10[0], d11[0] }; d11 = { d10[1], d11[1] } + * + * Unfortunately we need inline assembly for this: Instructions modifying two + * registers at once is not possible in GCC or Clang's IR, and they have to + * create a copy. + * + * aarch64 requires a different approach. + * + * In order to make it easier to write a decent compiler for aarch64, many + * quirks were removed, such as conditional execution. + * + * NEON was also affected by this. + * + * aarch64 cannot access the high bits of a Q-form register, and writes to a + * D-form register zero the high bits, similar to how writes to W-form scalar + * registers (or DWORD registers on x86_64) work. + * + * The formerly free vget_high intrinsics now require a vext (with a few + * exceptions) + * + * Additionally, VZIP was replaced by ZIP1 and ZIP2, which are the equivalent + * of PUNPCKL* and PUNPCKH* in SSE, respectively, in order to only modify one + * operand. + * + * The equivalent of the VZIP.32 on the lower and upper halves would be this + * mess: + * + * ext v2.4s, v0.4s, v0.4s, #2 // v2 = { v0[2], v0[3], v0[0], v0[1] } + * zip1 v1.2s, v0.2s, v2.2s // v1 = { v0[0], v2[0] } + * zip2 v0.2s, v0.2s, v1.2s // v0 = { v0[1], v2[1] } + * + * Instead, we use a literal downcast, vmovn_u64 (XTN), and vshrn_n_u64 (SHRN): + * + * shrn v1.2s, v0.2d, #32 // v1 = (uint32x2_t)(v0 >> 32); + * xtn v0.2s, v0.2d // v0 = (uint32x2_t)(v0 & 0xFFFFFFFF); + * + * This is available on ARMv7-A, but is less efficient than a single VZIP.32. + */ + +/*! + * Function-like macro: + * void XXH_SPLIT_IN_PLACE(uint64x2_t &in, uint32x2_t &outLo, uint32x2_t &outHi) + * { + * outLo = (uint32x2_t)(in & 0xFFFFFFFF); + * outHi = (uint32x2_t)(in >> 32); + * in = UNDEFINED; + * } + */ +# if !defined(XXH_NO_VZIP_HACK) /* define to disable */ \ + && (defined(__GNUC__) || defined(__clang__)) \ + && (defined(__arm__) || defined(__thumb__) || defined(_M_ARM)) +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + /* Undocumented GCC/Clang operand modifier: %e0 = lower D half, %f0 = upper D half */ \ + /* https://github.com/gcc-mirror/gcc/blob/38cf91e5/gcc/config/arm/arm.c#L22486 */ \ + /* https://github.com/llvm-mirror/llvm/blob/2c4ca683/lib/Target/ARM/ARMAsmPrinter.cpp#L399 */ \ + __asm__("vzip.32 %e0, %f0" : "+w" (in)); \ + (outLo) = vget_low_u32 (vreinterpretq_u32_u64(in)); \ + (outHi) = vget_high_u32(vreinterpretq_u32_u64(in)); \ + } while (0) +# else +# define XXH_SPLIT_IN_PLACE(in, outLo, outHi) \ + do { \ + (outLo) = vmovn_u64 (in); \ + (outHi) = vshrn_n_u64 ((in), 32); \ + } while (0) +# endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * On AArch64 when not optimizing for size, XXH3 will run 6 lanes using NEON and + * 2 lanes on scalar by default. + * + * This can be set to 2, 4, 6, or 8. ARMv7 will default to all 8 NEON lanes, as the + * emulated 64-bit arithmetic is too slow. + * + * Modern ARM CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but it can't + * have more than 2 NEON (F0/F1) micro-ops. If you are only using NEON instructions, + * you are only using 2/3 of the CPU bandwidth. + * + * This is even more noticeable on the more advanced cores like the A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, @ref XXH3_NEON_LANES lanes will be processed using NEON, and the + * remaining lanes will use scalar instructions. This improves the bandwidth + * and also gives the integer pipelines something to do besides twiddling loop + * counters and pointers. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__OPTIMIZE_SIZE__) +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +# if defined(__s390x__) +# include +# else +/* gcc's altivec.h can have the unwanted consequence to unconditionally + * #define bool, vector, and pixel keywords, + * with bad consequences for programs already using these keywords for other purposes. + * The paragraph defining these macros is skipped when __APPLE_ALTIVEC__ is defined. + * __APPLE_ALTIVEC__ is _generally_ defined automatically by the compiler, + * but it seems that, in some cases, it isn't. + * Force the build macro to be defined, so that keywords are not altered. + */ +# if defined(__GNUC__) && !defined(__APPLE_ALTIVEC__) +# define __APPLE_ALTIVEC__ +# endif +# include +# endif + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= 0x165667919E3779F9ULL; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= 0x9FB21C651E98DF25ULL; + h64 ^= (h64 >> 35) + len ; + h64 *= 0x9FB21C651E98DF25ULL; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); + + return XXH3_avalanche(acc); + } +} + +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + int const nbRounds = (int)len / 16; + int i; + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + acc = XXH3_avalanche(acc); + XXH_ASSERT(nbRounds >= 8); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + /* last bytes */ + acc += XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + return XXH3_avalanche(acc); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + __m512i const data_vec = _mm512_xor_si512 (acc_vec, shifted); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_shuffle_epi32 (data_key, (_MM_PERM_ENUM)_MM_SHUFFLE(0, 3, 0, 1)); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed = _mm512_mask_set1_epi64(_mm512_set1_epi64((xxh_i64)seed64), 0xAA, (xxh_i64)(0U - seed64)); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + /* GCC has a bug, _mm512_stream_load_si512 accepts 'void*', not 'void const*', + * this will warn "discards 'const' qualifier". */ + union { + const __m512i* cp; + void* p; + } remote_const_void; + remote_const_void.cp = src + i; + dest[i] = _mm512_add_epi64(_mm512_stream_load_si512(remote_const_void.p), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_stream_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_stream_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_stream_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_stream_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_stream_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_stream_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { + uint64x2_t* const xacc = (uint64x2_t *) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* const xinput = (const uint8_t *) input; + uint8_t const* const xsecret = (const uint8_t *) secret; + + size_t i; + /* NEON for the first few lanes (these loops are normally interleaved) */ + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint8x16_t data_vec = vld1q_u8(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8(xsecret + (i * 16)); + uint64x2_t data_key; + uint32x2_t data_key_lo, data_key_hi; + /* xacc[i] += swap(data_vec); */ + uint64x2_t const data64 = vreinterpretq_u64_u8(data_vec); + uint64x2_t const swapped = vextq_u64(data64, data64, 1); + xacc[i] = vaddq_u64 (xacc[i], swapped); + /* data_key = data_vec ^ key_vec; */ + data_key = vreinterpretq_u64_u8(veorq_u8(data_vec, key_vec)); + /* data_key_lo = (uint32x2_t) (data_key & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (data_key >> 32); + * data_key = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + /* xacc[i] += (uint64x2_t) data_key_lo * (uint64x2_t) data_key_hi; */ + xacc[i] = vmlal_u32 (xacc[i], data_key_lo, data_key_hi); + + } + /* Scalar for the remainder. This may be a zero iteration loop. */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { uint64x2_t* xacc = (uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + uint32x2_t prime = vdup_n_u32 (XXH_PRIME32_1); + + size_t i; + /* NEON for the first few lanes (these loops are normally interleaved) */ + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64 (acc_vec, 47); + uint64x2_t data_vec = veorq_u64 (acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint8x16_t key_vec = vld1q_u8 (xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64 (data_vec, vreinterpretq_u64_u8(key_vec)); + + /* xacc[i] *= XXH_PRIME32_1 */ + uint32x2_t data_key_lo, data_key_hi; + /* data_key_lo = (uint32x2_t) (xacc[i] & 0xFFFFFFFF); + * data_key_hi = (uint32x2_t) (xacc[i] >> 32); + * xacc[i] = UNDEFINED; */ + XXH_SPLIT_IN_PLACE(data_key, data_key_lo, data_key_hi); + { /* + * prod_hi = (data_key >> 32) * XXH_PRIME32_1; + * + * Avoid vmul_u32 + vshll_n_u32 since Clang 6 and 7 will + * incorrectly "optimize" this: + * tmp = vmul_u32(vmovn_u64(a), vmovn_u64(b)); + * shifted = vshll_n_u32(tmp, 32); + * to this: + * tmp = "vmulq_u64"(a, b); // no such thing! + * shifted = vshlq_n_u64(tmp, 32); + * + * However, unlike SSE, Clang lacks a 64-bit multiply routine + * for NEON, and it scalarizes two 64-bit multiplies instead. + * + * vmull_u32 has the same timing as vmul_u32, and it avoids + * this bug completely. + * See https://bugs.llvm.org/show_bug.cgi?id=39967 + */ + uint64x2_t prod_hi = vmull_u32 (data_key_hi, prime); + /* xacc[i] = prod_hi << 32; */ + xacc[i] = vshlq_n_u64(prod_hi, 32); + /* xacc[i] += (prod_hi & 0xFFFFFFFF) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(xacc[i], data_key_lo, prime); + } + } + /* Scalar for the remainder. This may be a zero iteration loop. */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + } +} + +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + unsigned int* const xacc = (unsigned int*) acc; + xxh_u64x2 const* const xinput = (xxh_u64x2 const*) input; /* no alignment restriction */ + xxh_u64x2 const* const xsecret = (xxh_u64x2 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = (xxh_u64x2)vec_xl(0, xacc + 4 * i); + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + /* xacc[i] = acc_vec; */ + vec_xst((xxh_u32x4)acc_vec, 0, xacc + 4 * i); + } +} + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_u64x2* const xacc = (xxh_u64x2*) acc; + const xxh_u64x2* const xsecret = (const xxh_u64x2*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +/* scalar variants - universal */ + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] += XXH_mult32to64(data_key & 0xFFFFFFFF, data_key >> 32); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__clang__) && defined(__aarch64__) + /* + * UGLY HACK: + * Clang generates a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes Clang to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + /* + * Note: in debug mode, this overrides the asm optimization + * and Clang will emit MOVK chains again. + */ + XXH_ASSERT(kSecretPtr == XXH3_kSecret); + + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes Clang to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate_512)(void* XXH_RESTRICT, const void*, const void*); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + + + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * XXH3_accumulate() + * Loops over XXH3_accumulate_512(). + * Assumption: nbStripes will not overflow the secret size + */ +XXH_FORCE_INLINE void +XXH3_accumulate( xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes, + XXH3_f_accumulate_512 f_acc512) +{ + size_t n; + for (n = 0; n < nbStripes; n++ ) { + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; + XXH_PREFETCH(in + XXH_PREFETCH_DIST); + f_acc512(acc, + in, + secret + n*XXH_SECRET_CONSUME_RATE); + } +} + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + XXH3_accumulate(acc, input + n*block_len, secret, nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + XXH3_accumulate(acc, input + nb_blocks*block_len, secret, nbStripes, f_acc512); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + f_acc512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* input, size_t len, + XXH64_hash_t seed, const xxh_u8* secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(const void* input, size_t len) +{ + return XXH3_64bits_internal(input, len, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, len, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, len, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ + +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH3_state_t* dst_state, const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/* Note : when XXH3_consumeStripes() is invoked, + * there must be a guarantee that at least one more byte must be consumed from input + * so that the function can blindly consume all stripes using the "normal" secret segment */ +XXH_FORCE_INLINE void +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ASSERT(nbStripes <= nbStripesPerBlock); /* can handle max 1 scramble per invocation */ + XXH_ASSERT(*nbStripesSoFarPtr < nbStripesPerBlock); + if (nbStripesPerBlock - *nbStripesSoFarPtr <= nbStripes) { + /* need a scrambling operation */ + size_t const nbStripesToEndofBlock = nbStripesPerBlock - *nbStripesSoFarPtr; + size_t const nbStripesAfterBlock = nbStripes - nbStripesToEndofBlock; + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripesToEndofBlock, f_acc512); + f_scramble(acc, secret + secretLimit); + XXH3_accumulate(acc, input + nbStripesToEndofBlock * XXH_STRIPE_LEN, secret, nbStripesAfterBlock, f_acc512); + *nbStripesSoFarPtr = nbStripesAfterBlock; + } else { + XXH3_accumulate(acc, input, secret + nbStripesSoFarPtr[0] * XXH_SECRET_CONSUME_RATE, nbStripes, f_acc512); + *nbStripesSoFarPtr += nbStripes; + } +} + +#ifndef XXH3_STREAM_USE_STACK +# ifndef __clang__ /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (state->bufferedSize + len <= XXH3_INTERNALBUFFER_SIZE) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + + /* large input to consume : ingest per full block */ + if ((size_t)(bEnd - input) > state->nbStripesPerBlock * XXH_STRIPE_LEN) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + XXH_ASSERT(state->nbStripesPerBlock >= state->nbStripesSoFar); + /* join to current block's end */ + { size_t const nbStripesToEnd = state->nbStripesPerBlock - state->nbStripesSoFar; + XXH_ASSERT(nbStripesToEnd <= nbStripes); + XXH3_accumulate(acc, input, secret + state->nbStripesSoFar * XXH_SECRET_CONSUME_RATE, nbStripesToEnd, f_acc512); + f_scramble(acc, secret + state->secretLimit); + state->nbStripesSoFar = 0; + input += nbStripesToEnd * XXH_STRIPE_LEN; + nbStripes -= nbStripesToEnd; + } + /* consume per entire blocks */ + while(nbStripes >= state->nbStripesPerBlock) { + XXH3_accumulate(acc, input, secret, state->nbStripesPerBlock, f_acc512); + f_scramble(acc, secret + state->secretLimit); + input += state->nbStripesPerBlock * XXH_STRIPE_LEN; + nbStripes -= state->nbStripesPerBlock; + } + /* consume last partial block */ + XXH3_accumulate(acc, input, secret, nbStripes, f_acc512); + input += nbStripes * XXH_STRIPE_LEN; + XXH_ASSERT(input < bEnd); /* at least some bytes left */ + state->nbStripesSoFar = nbStripes; + /* buffer predecessor of last partial stripe */ + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + XXH_ASSERT(bEnd - input <= XXH_STRIPE_LEN); + } else { + /* content to consume <= block size */ + /* Consume input by a multiple of internal buffer size */ + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + const xxh_u8* const limit = bEnd - XXH3_INTERNALBUFFER_SIZE; + do { + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc512, f_scramble); + input += XXH3_INTERNALBUFFER_SIZE; + } while (inputbuffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + } + } + + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate_512, XXH3_scrambleAcc); + /* last stripe */ + XXH3_accumulate_512(acc, + state->buffer + state->bufferedSize - XXH_STRIPE_LEN, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } else { /* bufferedSize < XXH_STRIPE_LEN */ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + XXH3_accumulate_512(acc, + lastStripe, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); + } +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= 0x9FB21C651E98DF25ULL; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + int const nbRounds = (int)len / 32; + int i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + for (i=0; i<4; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + (32 * i), + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + XXH_ASSERT(nbRounds >= 4); + for (i=4 ; i < nbRounds; i++) { + acc = XXH128_mix32B(acc, + input + (32 * i), + input + (32 * i) + 16, + secret + XXH3_MIDSIZE_STARTOFFSET + (32 * (i - 4)), + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + 0ULL - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc512, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate_512 f_acc512, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc512, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc512, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate_512, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(const void* input, size_t len, const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(const void* input, size_t len, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ + +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH3_state_t* statePtr, const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH3_state_t* state, const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate_512, XXH3_scrambleAcc); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} + +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API int XXH128_cmp(const void* h128_1, const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup xxh3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(void* secretBuffer, size_t secretSize, const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n +#include +#include + +#if defined(__GNUC__) && __GNUC__ >= 4 +# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l)) +#else +# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) memset((p),(v),(l)) +#endif + +#endif /* ZSTD_DEPS_COMMON */ + +/* Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#include + +#define ZSTD_malloc(s) malloc(s) +#define ZSTD_calloc(n,s) calloc((n), (s)) +#define ZSTD_free(p) free((p)) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor)) + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include +#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* Only requested when is known to be present. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +#include + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/zstd_internal.h b/External/Zstd/zstd-1.5.5/lib/common/zstd_internal.h new file mode 100644 index 000000000..1f942f27b --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/zstd_internal.h @@ -0,0 +1,392 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +#include "compiler.h" +#include "cpu.h" +#include "mem.h" +#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ +#ifndef ZSTD_NO_TRACE +# include "zstd_trace.h" +#else +# define ZSTD_TRACE 0 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 + +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1<= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ +typedef struct seqDef_s { + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ + U16 litLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ +} seqDef; + +/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} seqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/** + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->mlBase + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0x10000; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0x10000; + } + } + return seqLen; +} + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/External/Zstd/zstd-1.5.5/lib/common/zstd_trace.h b/External/Zstd/zstd-1.5.5/lib/common/zstd_trace.h new file mode 100644 index 000000000..da20534eb --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/common/zstd_trace.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_TRACE_H +#define ZSTD_TRACE_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include + +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386 and aarch64 + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 +#else +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR +#endif + +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif + +#if ZSTD_TRACE + +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; + +typedef struct { + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + unsigned streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + unsigned dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; + +/** + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). + * + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. + */ +typedef unsigned long long ZSTD_TraceCtx; + +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); + +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); + +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +#endif /* ZSTD_TRACE */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_TRACE_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/clevels.h b/External/Zstd/zstd-1.5.5/lib/compress/clevels.h new file mode 100644 index 000000000..c18da465f --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/clevels.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CLEVELS_H +#define ZSTD_CLEVELS_H + +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressionParameters */ +#include "../zstd.h" + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_MAX_CLEVEL 22 + +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" - for any srcSize > 256 KB */ + /* W, C, H, S, L, TL, strat */ + { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ + { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ + { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ + { 21, 16, 17, 1, 5, 0, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 0, ZSTD_dfast }, /* level 4 */ + { 21, 18, 19, 3, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6 */ + { 21, 19, 20, 4, 5, 8, ZSTD_lazy }, /* level 7 */ + { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 22, 23, 6, 5, 32, ZSTD_lazy2 }, /* level 12 */ + { 22, 22, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ + { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ + { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 18, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 18, 13, 14, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 18, 14, 14, 1, 5, 0, ZSTD_dfast }, /* level 2 */ + { 18, 16, 16, 1, 4, 0, ZSTD_dfast }, /* level 3 */ + { 18, 16, 17, 3, 5, 2, ZSTD_greedy }, /* level 4.*/ + { 18, 17, 18, 5, 5, 2, ZSTD_greedy }, /* level 5.*/ + { 18, 18, 19, 3, 5, 4, ZSTD_lazy }, /* level 6.*/ + { 18, 18, 19, 4, 4, 4, ZSTD_lazy }, /* level 7 */ + { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ + { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 17, 12, 13, 1, 6, 0, ZSTD_fast }, /* level 1 */ + { 17, 13, 15, 1, 5, 0, ZSTD_fast }, /* level 2 */ + { 17, 15, 16, 2, 5, 0, ZSTD_dfast }, /* level 3 */ + { 17, 17, 17, 2, 4, 0, ZSTD_dfast }, /* level 4 */ + { 17, 16, 17, 3, 4, 2, ZSTD_greedy }, /* level 5 */ + { 17, 16, 17, 3, 4, 4, ZSTD_lazy }, /* level 6 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 16, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 16, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 16, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ + { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ + { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ + { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ + { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ + { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ + { 14, 14, 15, 2, 4, 0, ZSTD_dfast }, /* level 3 */ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ + { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 8, 4, 8, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ + { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ + { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ +}, +}; + + + +#endif /* ZSTD_CLEVELS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/fse_compress.c b/External/Zstd/zstd-1.5.5/lib/compress/fse_compress.c new file mode 100644 index 000000000..5d3770808 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/fse_compress.c @@ -0,0 +1,624 @@ +/* ****************************************************************** + * FSE : Finite State Entropy encoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/compiler.h" +#include "../common/mem.h" /* U32, U16, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "hist.h" /* HIST_count_wksp */ +#include "../common/bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/error_private.h" +#define ZSTD_DEPS_NEED_MALLOC +#define ZSTD_DEPS_NEED_MATH64 +#include "../common/zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */ +#include "../common/bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 const maxSV1 = maxSymbolValue+1; + + U16* cumul = (U16*)workSpace; /* size = maxSV1 */ + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)(cumul + (maxSV1+1)); /* size = tableSize */ + + U32 highThreshold = tableSize-1; + + assert(((size_t)workSpace & 1) == 0); /* Must be 2 bytes-aligned */ + if (FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) > wkspSize) return ERROR(tableLog_tooLarge); + /* CTable header */ + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + assert(tableLog < 16); /* required for threshold strategy to work */ + + /* For explanations on how to distribute symbol values over the table : + * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + #ifdef __clang_analyzer__ + ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ + #endif + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u <= maxSV1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + assert(normalizedCounter[u-1] >= 0); + cumul[u] = cumul[u-1] + (U16)normalizedCounter[u-1]; + assert(cumul[u] >= cumul[u-1]); /* no overflow */ + } } + cumul[maxSV1] = (U16)(tableSize+1); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + /* Case for no low prob count symbols. Lay down 8 bytes at a time + * to reduce branch misses since we are operating on a small block + */ + BYTE* const spread = tableSymbol + tableSize; /* size = tableSize + 8 (may write beyond tableSize) */ + { U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Spread symbols across the table. Lack of lowprob symbols means that + * we don't need variable sized inner loop, so we can unroll the loop and + * reduce branch misses. + */ + { size_t position = 0; + size_t s; + size_t const unroll = 2; /* Experimentally determined optimal unroll */ + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableSymbol[uPosition] = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); /* Must have initialized all positions */ + } + } else { + U32 position = 0; + U32 symbol; + for (symbol=0; symbol highThreshold) + position = (position + step) & tableMask; /* Low proba area */ + } } + assert(position==0); /* Must have initialized all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u 1); + { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1); + U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut; + symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; + symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]); + total += (unsigned)normalizedCounter[s]; + } } } } + +#if 0 /* debug : symbol costs */ + DEBUGLOG(5, "\n --- table statistics : "); + { U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + DEBUGLOG(5, "%3u: w=%3i, maxBits=%u, fracBits=%.2f", + symbol, normalizedCounter[symbol], + FSE_getMaxNbBits(symbolTT, symbol), + (double)FSE_bitCost(symbolTT, tableLog, symbol, 8) / 256); + } } +#endif + + return 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/*-************************************************************** +* FSE NCount encoding +****************************************************************/ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog) +{ + size_t const maxHeaderSize = (((maxSymbolValue+1) * tableLog + + 4 /* bitCount initialized at 4 */ + + 2 /* first two symbols may use one additional bit each */) / 8) + + 1 /* round up to whole nb bytes */ + + 2 /* additional two bytes for bitstream flush */; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static size_t +FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream = 0; + int bitCount = 0; + unsigned symbol = 0; + unsigned const alphabetSize = maxSymbolValue + 1; + int previousIs0 = 0; + + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while ((symbol < alphabetSize) && (remaining>1)) { /* stops at 1 */ + if (previousIs0) { + unsigned start = symbol; + while ((symbol < alphabetSize) && !normalizedCounter[symbol]) symbol++; + if (symbol == alphabetSize) break; /* incorrect distribution */ + while (symbol >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (symbol >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (symbol-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { int count = normalizedCounter[symbol++]; + int const max = (2*threshold-1) - remaining; + remaining -= count < 0 ? -count : count; + count++; /* +1 for extra accuracy */ + if (count>=threshold) + count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; } + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + if (remaining != 1) + return ERROR(GENERIC); /* incorrect normalized distribution */ + assert(symbol <= alphabetSize); + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) + return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1 /* write in buffer is safe */); +} + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue, short lowProbCount) +{ + short const NOT_YET_ASSIGNED = -2; + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = lowProbCount; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + + norm[s]=NOT_YET_ASSIGNED; + } + ToDistribute = (1 << tableLog) - distributed; + + if (ToDistribute == 0) + return 0; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == NOT_YET_ASSIGNED) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) { maxV=s; maxC=count[s]; } + norm[maxV] += (short)ToDistribute; + return 0; + } + + if (total == 0) { + /* all of the symbols were low enough for the lowOne or lowThreshold */ + for (s=0; ToDistribute > 0; s = (s+1)%(maxSymbolValue+1)) + if (norm[s] > 0) { ToDistribute--; norm[s]++; } + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ZSTD_div64((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue, unsigned useLowProbCount) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { static U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + short const lowProbCount = useLowProbCount ? -1 : 1; + U64 const scale = 62 - tableLog; + U64 const step = ZSTD_div64((U64)1<<62, (U32)total); /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = lowProbCount; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) { largestP=proba; largest=s; } + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue, lowProbCount); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + RAWLOG(2, "%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U< FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/hist.c b/External/Zstd/zstd-1.5.5/lib/compress/hist.c new file mode 100644 index 000000000..e2fb431f0 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/hist.c @@ -0,0 +1,181 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/mem.h" /* U32, BYTE, etc. */ +#include "../common/debug.h" /* assert, DEBUGLOG */ +#include "../common/error_private.h" /* ERROR */ +#include "hist.h" + + +/* --- Error management --- */ +unsigned HIST_isError(size_t code) { return ERR_isError(code); } + +/*-************************************************************** + * Histogram functions + ****************************************************************/ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned largestCount=0; + + ZSTD_memset(count, 0, (maxSymbolValue+1) * sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip largestCount) largestCount = count[s]; + } + + return largestCount; +} + +typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; + +/* HIST_count_parallel_wksp() : + * store histogram into 4 intermediate tables, recombined at the end. + * this design makes better use of OoO cpus, + * and is noticeably faster when some values are heavily repeated. + * But it needs some additional workspace for intermediate tables. + * `workSpace` must be a U32 table of size >= HIST_WKSP_SIZE_U32. + * @return : largest histogram frequency, + * or an error code (notably when histogram's alphabet is larger than *maxSymbolValuePtr) */ +static size_t HIST_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + HIST_checkInput_e check, + U32* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + size_t const countSize = (*maxSymbolValuePtr + 1) * sizeof(*count); + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + /* safety checks */ + assert(*maxSymbolValuePtr <= 255); + if (!sourceSize) { + ZSTD_memset(count, 0, countSize); + *maxSymbolValuePtr = 0; + return 0; + } + ZSTD_memset(workSpace, 0, 4*256*sizeof(unsigned)); + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ip max) max = Counting1[s]; + } } + + { unsigned maxSymbolValue = 255; + while (!Counting1[maxSymbolValue]) maxSymbolValue--; + if (check && maxSymbolValue > *maxSymbolValuePtr) return ERROR(maxSymbolValue_tooSmall); + *maxSymbolValuePtr = maxSymbolValue; + ZSTD_memmove(count, Counting1, countSize); /* in case count & Counting1 are overlapping */ + } + return (size_t)max; +} + +/* HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if (sourceSize < 1500) /* heuristic threshold */ + return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); +} + +/* HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) +{ + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + if (*maxSymbolValuePtr < 255) + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); + *maxSymbolValuePtr = 255; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); +} + +#ifndef ZSTD_NO_UNUSED_FUNCTIONS +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); +} + +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + unsigned tmpCounters[HIST_WKSP_SIZE_U32]; + return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/compress/hist.h b/External/Zstd/zstd-1.5.5/lib/compress/hist.h new file mode 100644 index 000000000..887896b81 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/hist.h @@ -0,0 +1,75 @@ +/* ****************************************************************** + * hist : Histogram functions + * part of Finite State Entropy project + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* --- dependencies --- */ +#include "../common/zstd_deps.h" /* size_t */ + + +/* --- simple histogram functions --- */ + +/*! HIST_count(): + * Provides the precise count of each byte within a table 'count'. + * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + * Updates *maxSymbolValuePtr with actual largest symbol value detected. + * @return : count of the most frequent symbol (which isn't identified). + * or an error code, which can be tested using HIST_isError(). + * note : if return == srcSize, there is only one symbol. + */ +size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +unsigned HIST_isError(size_t code); /**< tells if a return value is an error code */ + + +/* --- advanced histogram functions --- */ + +#define HIST_WKSP_SIZE_U32 1024 +#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) +/** HIST_count_wksp() : + * Same as HIST_count(), but using an externally provided scratch buffer. + * Benefit is this function will use very little stack space. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/** HIST_countFast() : + * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` + */ +size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); + +/** HIST_countFast_wksp() : + * Same as HIST_countFast(), but using an externally provided scratch buffer. + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ +size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize, + void* workSpace, size_t workSpaceSize); + +/*! HIST_count_simple() : + * Same as HIST_countFast(), this function is unsafe, + * and will segfault if any value within `src` is `> *maxSymbolValuePtr`. + * It is also a bit slower for large inputs. + * However, it does not need any additional memory (not even on stack). + * @return : count of the most frequent symbol. + * Note this function doesn't produce any error (i.e. it must succeed). + */ +unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize); diff --git a/External/Zstd/zstd-1.5.5/lib/compress/huf_compress.c b/External/Zstd/zstd-1.5.5/lib/compress/huf_compress.c new file mode 100644 index 000000000..29871877a --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/huf_compress.c @@ -0,0 +1,1435 @@ +/* ****************************************************************** + * Huffman encoder, part of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" +#include "hist.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "../common/fse.h" /* header compression */ +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError +#define HUF_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Required declarations +****************************************************************/ +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + + +/* ************************************************************** +* Debug Traces +****************************************************************/ + +#if DEBUGLEVEL >= 2 + +static size_t showU32(const U32* arr, size_t size) +{ + size_t u; + for (u=0; u= add) { + assert(add < align); + assert(((size_t)aligned & mask) == 0); + *workspaceSizePtr -= add; + return aligned; + } else { + *workspaceSizePtr = 0; + return NULL; + } +} + + +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 + +typedef struct { + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + U32 scratchBuffer[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(HUF_TABLELOG_MAX, MAX_FSE_TABLELOG_FOR_HUFF_HEADER)]; + unsigned count[HUF_TABLELOG_MAX+1]; + S16 norm[HUF_TABLELOG_MAX+1]; +} HUF_CompressWeightsWksp; + +static size_t +HUF_compressWeights(void* dst, size_t dstSize, + const void* weightTable, size_t wtSize, + void* workspace, size_t workspaceSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + unsigned maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + HUF_CompressWeightsWksp* wksp = (HUF_CompressWeightsWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + + if (workspaceSize < sizeof(HUF_CompressWeightsWksp)) return ERROR(GENERIC); + + /* init conditions */ + if (wtSize <= 1) return 0; /* Not compressible */ + + /* Scan input and build symbol stats */ + { unsigned const maxCount = HIST_count_simple(wksp->count, &maxSymbolValue, weightTable, wtSize); /* never fails */ + if (maxCount == wtSize) return 1; /* only a single symbol in src : rle */ + if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(wksp->norm, tableLog, wksp->count, wtSize, maxSymbolValue, /* useLowProbCount */ 0) ); + + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, (size_t)(oend-op), wksp->norm, maxSymbolValue, tableLog) ); + op += hSize; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(wksp->CTable, wksp->norm, maxSymbolValue, tableLog, wksp->scratchBuffer, sizeof(wksp->scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, (size_t)(oend - op), weightTable, wtSize, wksp->CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + return (size_t)(op-ostart); +} + +static size_t HUF_getNbBits(HUF_CElt elt) +{ + return elt & 0xFF; +} + +static size_t HUF_getNbBitsFast(HUF_CElt elt) +{ + return elt; +} + +static size_t HUF_getValue(HUF_CElt elt) +{ + return elt & ~(size_t)0xFF; +} + +static size_t HUF_getValueFast(HUF_CElt elt) +{ + return elt; +} + +static void HUF_setNbBits(HUF_CElt* elt, size_t nbBits) +{ + assert(nbBits <= HUF_TABLELOG_ABSOLUTEMAX); + *elt = nbBits; +} + +static void HUF_setValue(HUF_CElt* elt, size_t value) +{ + size_t const nbBits = HUF_getNbBits(*elt); + if (nbBits > 0) { + assert((value >> nbBits) == 0); + *elt |= value << (sizeof(HUF_CElt) * 8 - nbBits); + } +} + +typedef struct { + HUF_CompressWeightsWksp wksp; + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; +} HUF_WriteCTableWksp; + +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, + const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, + void* workspace, size_t workspaceSize) +{ + HUF_CElt const* const ct = CTable + 1; + BYTE* op = (BYTE*)dst; + U32 n; + HUF_WriteCTableWksp* wksp = (HUF_WriteCTableWksp*)HUF_alignUpWorkspace(workspace, &workspaceSize, ZSTD_ALIGNOF(U32)); + + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE >= sizeof(HUF_WriteCTableWksp)); + + /* check conditions */ + if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + wksp->bitsToWeight[0] = 0; + for (n=1; nbitsToWeight[n] = (BYTE)(huffLog + 1 - n); + for (n=0; nhuffWeight[n] = wksp->bitsToWeight[HUF_getNbBits(ct[n])]; + + /* attempt weights compression by FSE */ + if (maxDstSize < 1) return ERROR(dstSize_tooSmall); + { CHECK_V_F(hSize, HUF_compressWeights(op+1, maxDstSize-1, wksp->huffWeight, maxSymbolValue, &wksp->wksp, sizeof(wksp->wksp)) ); + if ((hSize>1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; + } } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + wksp->huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; nhuffWeight[n] << 4) + wksp->huffWeight[n+1]); + return ((maxSymbolValue+1)/2) + 1; +} + + +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */ + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + HUF_CElt* const ct = CTable + 1; + + /* get symbol weights */ + CHECK_V_F(readSize, HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize)); + *hasZeroWeights = (rankVal[0] > 0); + + /* check result */ + if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > *maxSymbolValuePtr+1) return ERROR(maxSymbolValue_tooSmall); + + CTable[0] = tableLog; + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 curr = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = curr; + } } + + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n @targetNbBits + * to employ @targetNbBits instead. Then it adjusts the tree + * so that it remains a valid canonical Huffman tree. + * + * @pre The sum of the ranks of each symbol == 2^largestBits, + * where largestBits == huffNode[lastNonNull].nbBits. + * @post The sum of the ranks of each symbol == 2^largestBits, + * where largestBits is the return value (expected <= targetNbBits). + * + * @param huffNode The Huffman tree modified in place to enforce targetNbBits. + * It's presumed sorted, from most frequent to rarest symbol. + * @param lastNonNull The symbol with the lowest count in the Huffman tree. + * @param targetNbBits The allowed number of bits, which the Huffman tree + * may not respect. After this function the Huffman tree will + * respect targetNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment. + */ +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + /* early exit : no elt > targetNbBits, so the tree is already valid. */ + if (largestBits <= targetNbBits) return largestBits; + + DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits); + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - targetNbBits); + int n = (int)lastNonNull; + + /* Adjust any ranks > targetNbBits to targetNbBits. + * Compute totalCost, which is how far the sum of the ranks is + * we are over 2^largestBits after adjust the offending ranks. + */ + while (huffNode[n].nbBits > targetNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)targetNbBits; + n--; + } + /* n stops at huffNode[n].nbBits <= targetNbBits */ + assert(huffNode[n].nbBits <= targetNbBits); + /* n end at index of smallest symbol using < targetNbBits */ + while (huffNode[n].nbBits == targetNbBits) --n; + + /* renorm totalCost from 2^largestBits to 2^targetNbBits + * note : totalCost is necessarily a multiple of baseCost */ + assert(((U32)totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - targetNbBits); + assert(totalCost > 0); + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + + /* Get pos of last (smallest = lowest cum. count) symbol per rank */ + ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = targetNbBits; + int pos; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */ + rankLast[targetNbBits-currentNbBits] = (U32)pos; + } } + + while (totalCost > 0) { + /* Try to reduce the next power of 2 above totalCost because we + * gain back half the rank. + */ + U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 const highPos = rankLast[nBitsToDecrease]; + U32 const lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + /* Decrease highPos if no symbols of lowPos or if it is + * not cheaper to remove 2 lowPos than highPos. + */ + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + assert(rankLast[nBitsToDecrease] != noSymbol || nBitsToDecrease == 1); + /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) + nBitsToDecrease++; + assert(rankLast[nBitsToDecrease] != noSymbol); + /* Increase the number of bits to gain back half the rank cost. */ + totalCost -= 1 << (nBitsToDecrease-1); + huffNode[rankLast[nBitsToDecrease]].nbBits++; + + /* Fix up the new rank. + * If the new rank was empty, this symbol is now its smallest. + * Otherwise, this symbol will be the largest in the new rank so no adjustment. + */ + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; + /* Fix up the old rank. + * If the symbol was at position 0, meaning it was the highest weight symbol in the tree, + * it must be the only symbol in its rank, so the old rank now has no symbols. + * Otherwise, since the Huffman nodes are sorted by count, the previous position is now + * the smallest node in the rank. If the previous position belongs to a different rank, + * then the rank is now empty. + */ + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } + } /* while (totalCost > 0) */ + + /* If we've removed too much weight, then we have to add it back. + * To avoid overshooting again, we only adjust the smallest rank. + * We take the largest nodes from the lowest rank 0 and move them + * to rank 1. There's guaranteed to be enough rank 0 symbols because + * TODO. + */ + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + /* special case : no rank 1 symbol (using targetNbBits-1); + * let's create one from largest rank 0 (using targetNbBits). + */ + if (rankLast[1] == noSymbol) { + while (huffNode[n].nbBits == targetNbBits) n--; + huffNode[n+1].nbBits--; + assert(n >= 0); + rankLast[1] = (U32)(n+1); + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } + } /* repay normalized cost */ + } /* there are several too large elements (at least >= 2) */ + + return targetNbBits; +} + +typedef struct { + U16 base; + U16 curr; +} rankPos; + +typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)]; + +/* Number of buckets available for HUF_sort() */ +#define RANK_POSITION_TABLE_SIZE 192 + +typedef struct { + huffNodeTable huffNodeTbl; + rankPos rankPosition[RANK_POSITION_TABLE_SIZE]; +} HUF_buildCTable_wksp_tables; + +/* RANK_POSITION_DISTINCT_COUNT_CUTOFF == Cutoff point in HUF_sort() buckets for which we use log2 bucketing. + * Strategy is to use as many buckets as possible for representing distinct + * counts while using the remainder to represent all "large" counts. + * + * To satisfy this requirement for 192 buckets, we can do the following: + * Let buckets 0-166 represent distinct counts of [0, 166] + * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. + */ +#define RANK_POSITION_MAX_COUNT_LOG 32 +#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */) +#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */) + +/* Return the appropriate bucket index for a given count. See definition of + * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy. + */ +static U32 HUF_getIndex(U32 const count) { + return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF) + ? count + : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; +} + +/* Helper swap function for HUF_quickSortPartition() */ +static void HUF_swapNodes(nodeElt* a, nodeElt* b) { + nodeElt tmp = *a; + *a = *b; + *b = tmp; +} + +/* Returns 0 if the huffNode array is not sorted by descending count */ +MEM_STATIC int HUF_isSorted(nodeElt huffNode[], U32 const maxSymbolValue1) { + U32 i; + for (i = 1; i < maxSymbolValue1; ++i) { + if (huffNode[i].count > huffNode[i-1].count) { + return 0; + } + } + return 1; +} + +/* Insertion sort by descending order */ +HINT_INLINE void HUF_insertionSort(nodeElt huffNode[], int const low, int const high) { + int i; + int const size = high-low+1; + huffNode += low; + for (i = 1; i < size; ++i) { + nodeElt const key = huffNode[i]; + int j = i - 1; + while (j >= 0 && huffNode[j].count < key.count) { + huffNode[j + 1] = huffNode[j]; + j--; + } + huffNode[j + 1] = key; + } +} + +/* Pivot helper function for quicksort. */ +static int HUF_quickSortPartition(nodeElt arr[], int const low, int const high) { + /* Simply select rightmost element as pivot. "Better" selectors like + * median-of-three don't experimentally appear to have any benefit. + */ + U32 const pivot = arr[high].count; + int i = low - 1; + int j = low; + for ( ; j < high; j++) { + if (arr[j].count > pivot) { + i++; + HUF_swapNodes(&arr[i], &arr[j]); + } + } + HUF_swapNodes(&arr[i + 1], &arr[high]); + return i + 1; +} + +/* Classic quicksort by descending with partially iterative calls + * to reduce worst case callstack size. + */ +static void HUF_simpleQuickSort(nodeElt arr[], int low, int high) { + int const kInsertionSortThreshold = 8; + if (high - low < kInsertionSortThreshold) { + HUF_insertionSort(arr, low, high); + return; + } + while (low < high) { + int const idx = HUF_quickSortPartition(arr, low, high); + if (idx - low < high - idx) { + HUF_simpleQuickSort(arr, low, idx - 1); + low = idx + 1; + } else { + HUF_simpleQuickSort(arr, idx + 1, high); + high = idx - 1; + } + } +} + +/** + * HUF_sort(): + * Sorts the symbols [0, maxSymbolValue] by count[symbol] in decreasing order. + * This is a typical bucket sorting strategy that uses either quicksort or insertion sort to sort each bucket. + * + * @param[out] huffNode Sorted symbols by decreasing count. Only members `.count` and `.byte` are filled. + * Must have (maxSymbolValue + 1) entries. + * @param[in] count Histogram of the symbols. + * @param[in] maxSymbolValue Maximum symbol value. + * @param rankPosition This is a scratch workspace. Must have RANK_POSITION_TABLE_SIZE entries. + */ +static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSymbolValue, rankPos rankPosition[]) { + U32 n; + U32 const maxSymbolValue1 = maxSymbolValue+1; + + /* Compute base and set curr to base. + * For symbol s let lowerRank = HUF_getIndex(count[n]) and rank = lowerRank + 1. + * See HUF_getIndex to see bucketing strategy. + * We attribute each symbol to lowerRank's base value, because we want to know where + * each rank begins in the output, so for rank R we want to count ranks R+1 and above. + */ + ZSTD_memset(rankPosition, 0, sizeof(*rankPosition) * RANK_POSITION_TABLE_SIZE); + for (n = 0; n < maxSymbolValue1; ++n) { + U32 lowerRank = HUF_getIndex(count[n]); + assert(lowerRank < RANK_POSITION_TABLE_SIZE - 1); + rankPosition[lowerRank].base++; + } + + assert(rankPosition[RANK_POSITION_TABLE_SIZE - 1].base == 0); + /* Set up the rankPosition table */ + for (n = RANK_POSITION_TABLE_SIZE - 1; n > 0; --n) { + rankPosition[n-1].base += rankPosition[n].base; + rankPosition[n-1].curr = rankPosition[n-1].base; + } + + /* Insert each symbol into their appropriate bucket, setting up rankPosition table. */ + for (n = 0; n < maxSymbolValue1; ++n) { + U32 const c = count[n]; + U32 const r = HUF_getIndex(c) + 1; + U32 const pos = rankPosition[r].curr++; + assert(pos < maxSymbolValue1); + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } + + /* Sort each bucket. */ + for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) { + int const bucketSize = rankPosition[n].curr - rankPosition[n].base; + U32 const bucketStartIdx = rankPosition[n].base; + if (bucketSize > 1) { + assert(bucketStartIdx < maxSymbolValue1); + HUF_simpleQuickSort(huffNode + bucketStartIdx, 0, bucketSize-1); + } + } + + assert(HUF_isSorted(huffNode, maxSymbolValue1)); +} + + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) + +/* HUF_buildTree(): + * Takes the huffNode array sorted by HUF_sort() and builds an unlimited-depth Huffman tree. + * + * @param huffNode The array sorted by HUF_sort(). Builds the Huffman tree in this array. + * @param maxSymbolValue The maximum symbol value. + * @return The smallest node in the Huffman tree (by count). + */ +static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) +{ + nodeElt* const huffNode0 = huffNode - 1; + int nonNullRank; + int lowS, lowN; + int nodeNb = STARTNODE; + int n, nodeRoot; + DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1); + /* init for parents */ + nonNullRank = (int)maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = (U16)nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + int const n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + int const n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = (U16)nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1)); + + return nonNullRank; +} + +/** + * HUF_buildCTableFromTree(): + * Build the CTable given the Huffman tree in huffNode. + * + * @param[out] CTable The output Huffman CTable. + * @param huffNode The Huffman tree. + * @param nonNullRank The last and smallest node in the Huffman tree. + * @param maxSymbolValue The maximum symbol value. + * @param maxNbBits The exact maximum number of bits used in the Huffman tree. + */ +static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, int nonNullRank, U32 maxSymbolValue, U32 maxNbBits) +{ + HUF_CElt* const ct = CTable + 1; + /* fill result into ctable (val, nbBits) */ + int n; + U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + int const alphabetSize = (int)(maxSymbolValue + 1); + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine starting value per rank */ + { U16 min = 0; + for (n=(int)maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; nhuffNodeTbl; + nodeElt* const huffNode = huffNode0+1; + int nonNullRank; + + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables)); + + DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1); + + /* safety checks */ + if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) + return ERROR(workSpace_tooSmall); + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) + return ERROR(maxSymbolValue_tooLarge); + ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1)); + + /* build tree */ + nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); + + /* determine and enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + + HUF_buildCTableFromTree(CTable, huffNode, nonNullRank, maxSymbolValue, maxNbBits); + + return maxNbBits; +} + +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) +{ + HUF_CElt const* ct = CTable + 1; + size_t nbBits = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + nbBits += HUF_getNbBits(ct[s]) * count[s]; + } + return nbBits >> 3; +} + +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue) { + HUF_CElt const* ct = CTable + 1; + int bad = 0; + int s; + for (s = 0; s <= (int)maxSymbolValue; ++s) { + bad |= (count[s] != 0) & (HUF_getNbBits(ct[s]) == 0); + } + return !bad; +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +/** HUF_CStream_t: + * Huffman uses its own BIT_CStream_t implementation. + * There are three major differences from BIT_CStream_t: + * 1. HUF_addBits() takes a HUF_CElt (size_t) which is + * the pair (nbBits, value) in the format: + * format: + * - Bits [0, 4) = nbBits + * - Bits [4, 64 - nbBits) = 0 + * - Bits [64 - nbBits, 64) = value + * 2. The bitContainer is built from the upper bits and + * right shifted. E.g. to add a new value of N bits + * you right shift the bitContainer by N, then or in + * the new value into the N upper bits. + * 3. The bitstream has two bit containers. You can add + * bits to the second container and merge them into + * the first container. + */ + +#define HUF_BITS_IN_CONTAINER (sizeof(size_t) * 8) + +typedef struct { + size_t bitContainer[2]; + size_t bitPos[2]; + + BYTE* startPtr; + BYTE* ptr; + BYTE* endPtr; +} HUF_CStream_t; + +/**! HUF_initCStream(): + * Initializes the bitstream. + * @returns 0 or an error code. + */ +static size_t HUF_initCStream(HUF_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + ZSTD_memset(bitC, 0, sizeof(*bitC)); + bitC->startPtr = (BYTE*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer[0]); + if (dstCapacity <= sizeof(bitC->bitContainer[0])) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! HUF_addBits(): + * Adds the symbol stored in HUF_CElt elt to the bitstream. + * + * @param elt The element we're adding. This is a (nbBits, value) pair. + * See the HUF_CStream_t docs for the format. + * @param idx Insert into the bitstream at this idx. + * @param kFast This is a template parameter. If the bitstream is guaranteed + * to have at least 4 unused bits after this call it may be 1, + * otherwise it must be 0. HUF_addBits() is faster when fast is set. + */ +FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int idx, int kFast) +{ + assert(idx <= 1); + assert(HUF_getNbBits(elt) <= HUF_TABLELOG_ABSOLUTEMAX); + /* This is efficient on x86-64 with BMI2 because shrx + * only reads the low 6 bits of the register. The compiler + * knows this and elides the mask. When fast is set, + * every operation can use the same value loaded from elt. + */ + bitC->bitContainer[idx] >>= HUF_getNbBits(elt); + bitC->bitContainer[idx] |= kFast ? HUF_getValueFast(elt) : HUF_getValue(elt); + /* We only read the low 8 bits of bitC->bitPos[idx] so it + * doesn't matter that the high bits have noise from the value. + */ + bitC->bitPos[idx] += HUF_getNbBitsFast(elt); + assert((bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + /* The last 4-bits of elt are dirty if fast is set, + * so we must not be overwriting bits that have already been + * inserted into the bit container. + */ +#if DEBUGLEVEL >= 1 + { + size_t const nbBits = HUF_getNbBits(elt); + size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1; + (void)dirtyBits; + /* Middle bits are 0. */ + assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); + /* We didn't overwrite any bits in the bit container. */ + assert(!kFast || (bitC->bitPos[idx] & 0xFF) <= HUF_BITS_IN_CONTAINER); + (void)dirtyBits; + } +#endif +} + +FORCE_INLINE_TEMPLATE void HUF_zeroIndex1(HUF_CStream_t* bitC) +{ + bitC->bitContainer[1] = 0; + bitC->bitPos[1] = 0; +} + +/*! HUF_mergeIndex1() : + * Merges the bit container @ index 1 into the bit container @ index 0 + * and zeros the bit container @ index 1. + */ +FORCE_INLINE_TEMPLATE void HUF_mergeIndex1(HUF_CStream_t* bitC) +{ + assert((bitC->bitPos[1] & 0xFF) < HUF_BITS_IN_CONTAINER); + bitC->bitContainer[0] >>= (bitC->bitPos[1] & 0xFF); + bitC->bitContainer[0] |= bitC->bitContainer[1]; + bitC->bitPos[0] += bitC->bitPos[1]; + assert((bitC->bitPos[0] & 0xFF) <= HUF_BITS_IN_CONTAINER); +} + +/*! HUF_flushBits() : +* Flushes the bits in the bit container @ index 0. +* +* @post bitPos will be < 8. +* @param kFast If kFast is set then we must know a-priori that +* the bit container will not overflow. +*/ +FORCE_INLINE_TEMPLATE void HUF_flushBits(HUF_CStream_t* bitC, int kFast) +{ + /* The upper bits of bitPos are noisy, so we must mask by 0xFF. */ + size_t const nbBits = bitC->bitPos[0] & 0xFF; + size_t const nbBytes = nbBits >> 3; + /* The top nbBits bits of bitContainer are the ones we need. */ + size_t const bitContainer = bitC->bitContainer[0] >> (HUF_BITS_IN_CONTAINER - nbBits); + /* Mask bitPos to account for the bytes we consumed. */ + bitC->bitPos[0] &= 7; + assert(nbBits > 0); + assert(nbBits <= sizeof(bitC->bitContainer[0]) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitContainer); + bitC->ptr += nbBytes; + assert(!kFast || bitC->ptr <= bitC->endPtr); + if (!kFast && bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + /* bitContainer doesn't need to be modified because the leftover + * bits are already the top bitPos bits. And we don't care about + * noise in the lower values. + */ +} + +/*! HUF_endMark() + * @returns The Huffman stream end mark: A 1-bit value = 1. + */ +static HUF_CElt HUF_endMark(void) +{ + HUF_CElt endMark; + HUF_setNbBits(&endMark, 1); + HUF_setValue(&endMark, 1); + return endMark; +} + +/*! HUF_closeCStream() : + * @return Size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +static size_t HUF_closeCStream(HUF_CStream_t* bitC) +{ + HUF_addBits(bitC, HUF_endMark(), /* idx */ 0, /* kFast */ 0); + HUF_flushBits(bitC, /* kFast */ 0); + { + size_t const nbBits = bitC->bitPos[0] & 0xFF; + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0); + } +} + +FORCE_INLINE_TEMPLATE void +HUF_encodeSymbol(HUF_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable, int idx, int fast) +{ + HUF_addBits(bitCPtr, CTable[symbol], idx, fast); +} + +FORCE_INLINE_TEMPLATE void +HUF_compress1X_usingCTable_internal_body_loop(HUF_CStream_t* bitC, + const BYTE* ip, size_t srcSize, + const HUF_CElt* ct, + int kUnroll, int kFastFlush, int kLastFast) +{ + /* Join to kUnroll */ + int n = (int)srcSize; + int rem = n % kUnroll; + if (rem > 0) { + for (; rem > 0; --rem) { + HUF_encodeSymbol(bitC, ip[--n], ct, 0, /* fast */ 0); + } + HUF_flushBits(bitC, kFastFlush); + } + assert(n % kUnroll == 0); + + /* Join to 2 * kUnroll */ + if (n % (2 * kUnroll)) { + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, 0, 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, 0, kLastFast); + HUF_flushBits(bitC, kFastFlush); + n -= kUnroll; + } + assert(n % (2 * kUnroll) == 0); + + for (; n>0; n-= 2 * kUnroll) { + /* Encode kUnroll symbols into the bitstream @ index 0. */ + int u; + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - u], ct, /* idx */ 0, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll], ct, /* idx */ 0, /* fast */ kLastFast); + HUF_flushBits(bitC, kFastFlush); + /* Encode kUnroll symbols into the bitstream @ index 1. + * This allows us to start filling the bit container + * without any data dependencies. + */ + HUF_zeroIndex1(bitC); + for (u = 1; u < kUnroll; ++u) { + HUF_encodeSymbol(bitC, ip[n - kUnroll - u], ct, /* idx */ 1, /* fast */ 1); + } + HUF_encodeSymbol(bitC, ip[n - kUnroll - kUnroll], ct, /* idx */ 1, /* fast */ kLastFast); + /* Merge bitstream @ index 1 into the bitstream @ index 0 */ + HUF_mergeIndex1(bitC); + HUF_flushBits(bitC, kFastFlush); + } + assert(n == 0); + +} + +/** + * Returns a tight upper bound on the output space needed by Huffman + * with 8 bytes buffer to handle over-writes. If the output is at least + * this large we don't need to do bounds checks during Huffman encoding. + */ +static size_t HUF_tightCompressBound(size_t srcSize, size_t tableLog) +{ + return ((srcSize * tableLog) >> 3) + 8; +} + + +FORCE_INLINE_TEMPLATE size_t +HUF_compress1X_usingCTable_internal_body(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + U32 const tableLog = (U32)CTable[0]; + HUF_CElt const* ct = CTable + 1; + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + HUF_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const initErr = HUF_initCStream(&bitC, op, (size_t)(oend-op)); + if (HUF_isError(initErr)) return 0; } + + if (dstSize < HUF_tightCompressBound(srcSize, (size_t)tableLog) || tableLog > 11) + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ MEM_32bits() ? 2 : 4, /* kFast */ 0, /* kLastFast */ 0); + else { + if (MEM_32bits()) { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: ZSTD_FALLTHROUGH; + case 9: ZSTD_FALLTHROUGH; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 2, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 7: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 3, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } else { + switch (tableLog) { + case 11: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 10: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 5, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + case 9: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 6, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 8: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 7, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 7: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 8, /* kFastFlush */ 1, /* kLastFast */ 0); + break; + case 6: ZSTD_FALLTHROUGH; + default: + HUF_compress1X_usingCTable_internal_body_loop(&bitC, ip, srcSize, ct, /* kUnroll */ 9, /* kFastFlush */ 1, /* kLastFast */ 1); + break; + } + } + } + assert(bitC.ptr <= bitC.endPtr); + + return HUF_closeCStream(&bitC); +} + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +HUF_compress1X_usingCTable_internal_bmi2(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable) +{ + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int flags) +{ + if (flags & HUF_flags_bmi2) { + return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); + } + return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); +} + +#else + +static size_t +HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, const int flags) +{ + (void)flags; + return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); +} + +#endif + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) +{ + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); +} + +static size_t +HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, + const void* src, size_t srcSize, + const HUF_CElt* CTable, int flags) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + assert(op <= oend); + assert(ip <= iend); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) ); + if (cSize == 0 || cSize > 65535) return 0; + op += cSize; + } + + return (size_t)(op-ostart); +} + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) +{ + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); +} + +typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; + +static size_t HUF_compressCTable_internal( + BYTE* const ostart, BYTE* op, BYTE* const oend, + const void* src, size_t srcSize, + HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags) +{ + size_t const cSize = (nbStreams==HUF_singleStream) ? + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags); + if (HUF_isError(cSize)) { return cSize; } + if (cSize==0) { return 0; } /* uncompressible */ + op += cSize; + /* check compressibility */ + assert(op >= ostart); + if ((size_t)(op-ostart) >= srcSize-1) { return 0; } + return (size_t)(op-ostart); +} + +typedef struct { + unsigned count[HUF_SYMBOLVALUE_MAX + 1]; + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(HUF_SYMBOLVALUE_MAX)]; + union { + HUF_buildCTable_wksp_tables buildCTable_wksp; + HUF_WriteCTableWksp writeCTable_wksp; + U32 hist_wksp[HIST_WKSP_SIZE_U32]; + } wksps; +} HUF_compress_tables_t; + +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 +#define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ + +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue) +{ + unsigned cardinality = 0; + unsigned i; + + for (i = 0; i < maxSymbolValue + 1; i++) { + if (count[i] != 0) cardinality += 1; + } + + return cardinality; +} + +unsigned HUF_minTableLog(unsigned symbolCardinality) +{ + U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1; + return minBitsSymbols; +} + +unsigned HUF_optimalTableLog( + unsigned maxTableLog, + size_t srcSize, + unsigned maxSymbolValue, + void* workSpace, size_t wkspSize, + HUF_CElt* table, + const unsigned* count, + int flags) +{ + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables)); + + if (!(flags & HUF_flags_optimalDepth)) { + /* cheap evaluation, based on FSE */ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + } + + { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); + size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); + size_t maxBits, hSize, newSize; + const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); + const unsigned minTableLog = HUF_minTableLog(symbolCardinality); + size_t optSize = ((size_t) ~0) - 1; + unsigned optLog = maxTableLog, optLogGuess; + + DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); + + /* Search until size increases */ + for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { + DEBUGLOG(7, "checking for huffLog=%u", optLogGuess); + maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); + if (ERR_isError(maxBits)) continue; + + if (maxBits < optLogGuess && optLogGuess > minTableLog) break; + + hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize); + + if (ERR_isError(hSize)) continue; + + newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; + + if (newSize > optSize + 1) { + break; + } + + if (newSize < optSize) { + optSize = newSize; + optLog = optLogGuess; + } + } + assert(optLog <= HUF_TABLELOG_MAX); + return optLog; + } +} + +/* HUF_compress_internal() : + * `workSpace_align4` must be aligned on 4-bytes boundaries, + * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ +static size_t +HUF_compress_internal (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + HUF_nbStreams_e nbStreams, + void* workSpace, size_t wkspSize, + HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags) +{ + HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize); + HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); + + /* checks & inits */ + if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall); + if (!srcSize) return 0; /* Uncompressed */ + if (!dstSize) return 0; /* cannot fit anything within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Heuristic : If old table is valid, use it for small inputs */ + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } + + /* If uncompressible data is suspected, do a smaller sampling first */ + DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); + if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { + size_t largestTotal = 0; + DEBUGLOG(5, "input suspected incompressible : sampling to check"); + { unsigned maxSymbolValueBegin = maxSymbolValue; + CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestBegin; + } + { unsigned maxSymbolValueEnd = maxSymbolValue; + CHECK_V_F(largestEnd, HIST_count_simple (table->count, &maxSymbolValueEnd, (const BYTE*)src + srcSize - SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); + largestTotal += largestEnd; + } + if (largestTotal <= ((2 * SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->wksps.hist_wksp, sizeof(table->wksps.hist_wksp)) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ + } + DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1)); + + /* Check validity of previous table */ + if ( repeat + && *repeat == HUF_repeat_check + && !HUF_validateCTable(oldHufTable, table->count, maxSymbolValue)) { + *repeat = HUF_repeat_none; + } + /* Heuristic : use existing table for small inputs */ + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags); + { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, + maxSymbolValue, huffLog, + &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); + CHECK_F(maxBits); + huffLog = (U32)maxBits; + DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1)); + } + /* Zero unused symbols in CTable, so we can check it for validity */ + { + size_t const ctableSize = HUF_CTABLE_SIZE_ST(maxSymbolValue); + size_t const unusedSize = sizeof(table->CTable) - ctableSize * sizeof(HUF_CElt); + ZSTD_memset(table->CTable + ctableSize, 0, unusedSize); + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable_wksp(op, dstSize, table->CTable, maxSymbolValue, huffLog, + &table->wksps.writeCTable_wksp, sizeof(table->wksps.writeCTable_wksp)) ); + /* Check if using previous huffman table is beneficial */ + if (repeat && *repeat != HUF_repeat_none) { + size_t const oldSize = HUF_estimateCompressedSize(oldHufTable, table->count, maxSymbolValue); + size_t const newSize = HUF_estimateCompressedSize(table->CTable, table->count, maxSymbolValue); + if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, oldHufTable, flags); + } } + + /* Use the new huffman table */ + if (hSize + 12ul >= srcSize) { return 0; } + op += hSize; + if (repeat) { *repeat = HUF_repeat_none; } + if (oldHufTable) + ZSTD_memcpy(oldHufTable, table->CTable, sizeof(table->CTable)); /* Save new table */ + } + return HUF_compressCTable_internal(ostart, op, oend, + src, srcSize, + nbStreams, table->CTable, flags); +} + +size_t HUF_compress1X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) +{ + DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize); + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_singleStream, + workSpace, wkspSize, hufTable, + repeat, flags); +} + +/* HUF_compress4X_repeat(): + * compress input using 4 streams. + * consider skipping quickly + * re-use an existing huffman compression table */ +size_t HUF_compress4X_repeat (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize, + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) +{ + DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize); + return HUF_compress_internal(dst, dstSize, src, srcSize, + maxSymbolValue, huffLog, HUF_fourStreams, + workSpace, wkspSize, + hufTable, repeat, flags); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress.c new file mode 100644 index 000000000..d6133e70b --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress.c @@ -0,0 +1,7032 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ +#include "../common/mem.h" +#include "hist.h" /* HIST_countFast_wksp */ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "../common/fse.h" +#include "../common/huf.h" +#include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" +#include "zstd_fast.h" +#include "zstd_double_fast.h" +#include "zstd_lazy.h" +#include "zstd_opt.h" +#include "zstd_ldm.h" +#include "zstd_compress_superblock.h" +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_rotateRight_U64 */ + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * COMPRESS_HEAPMODE : + * Select how default decompression function ZSTD_compress() allocates its context, + * on stack (0, default), or into heap (1). + * Note that functions with explicit context such as ZSTD_compressCCtx() are unaffected. + */ +#ifndef ZSTD_COMPRESS_HEAPMODE +# define ZSTD_COMPRESS_HEAPMODE 0 +#endif + +/*! + * ZSTD_HASHLOG3_MAX : + * Maximum size of the hash table dedicated to find 3-bytes matches, + * in log format, aka 17 => 1 << 17 == 128Ki positions. + * This structure is only used in zstd_opt. + * Since allocation is centralized for all strategies, it has to be known here. + * The actual (selected) size of the hash table is then stored in ZSTD_matchState_t.hashLog3, + * so that zstd_opt.c doesn't need to know about this constant. + */ +#ifndef ZSTD_HASHLOG3_MAX +# define ZSTD_HASHLOG3_MAX 17 +#endif + +/*-************************************* +* Helper functions +***************************************/ +/* ZSTD_compressBound() + * Note that the result from this function is only valid for + * the one-pass compression functions. + * When employing the streaming mode, + * if flushes are frequently altering the size of blocks, + * the overhead from block headers can make the compressed data larger + * than the return value of ZSTD_compressBound(). + */ +size_t ZSTD_compressBound(size_t srcSize) { + size_t const r = ZSTD_COMPRESSBOUND(srcSize); + if (r==0) return ERROR(srcSize_wrong); + return r; +} + + +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CDict_s { + const void* dictContent; + size_t dictContentSize; + ZSTD_dictContentType_e dictContentType; /* The dictContentType the CDict was created with */ + U32* entropyWorkspace; /* entropy workspace of HUF_WORKSPACE_SIZE bytes */ + ZSTD_cwksp workspace; + ZSTD_matchState_t matchState; + ZSTD_compressedBlockState_t cBlockState; + ZSTD_customMem customMem; + U32 dictID; + int compressionLevel; /* 0 indicates that advanced API was used to select CDict params */ + ZSTD_paramSwitch_e useRowMatchFinder; /* Indicates whether the CDict was created with params that would use + * row-based matchfinder. Unless the cdict is reloaded, we will use + * the same greedy/lazy matchfinder at compression time. + */ +}; /* typedef'd to ZSTD_CDict within "zstd.h" */ + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(ZSTD_defaultCMem); +} + +static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) +{ + assert(cctx != NULL); + ZSTD_memset(cctx, 0, sizeof(*cctx)); + cctx->customMem = memManager; + cctx->bmi2 = ZSTD_cpuSupportsBmi2(); + { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_STATIC_ASSERT(zcss_init==0); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN==(0ULL - 1)); + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + { ZSTD_CCtx* const cctx = (ZSTD_CCtx*)ZSTD_customMalloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + ZSTD_initCCtx(cctx, customMem); + return cctx; + } +} + +ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize) +{ + ZSTD_cwksp ws; + ZSTD_CCtx* cctx; + if (workspaceSize <= sizeof(ZSTD_CCtx)) return NULL; /* minimum size */ + if ((size_t)workspace & 7) return NULL; /* must be 8-aligned */ + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + + cctx = (ZSTD_CCtx*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CCtx)); + if (cctx == NULL) return NULL; + + ZSTD_memset(cctx, 0, sizeof(ZSTD_CCtx)); + ZSTD_cwksp_move(&cctx->workspace, &ws); + cctx->staticSize = workspaceSize; + + /* statically sized space. entropyWorkspace never moves (but prev/next block swap places) */ + if (!ZSTD_cwksp_check_available(&cctx->workspace, ENTROPY_WORKSPACE_SIZE + 2 * sizeof(ZSTD_compressedBlockState_t))) return NULL; + cctx->blockState.prevCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->blockState.nextCBlock = (ZSTD_compressedBlockState_t*)ZSTD_cwksp_reserve_object(&cctx->workspace, sizeof(ZSTD_compressedBlockState_t)); + cctx->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cctx->workspace, ENTROPY_WORKSPACE_SIZE); + cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); + return cctx; +} + +/** + * Clears and frees all of the dictionaries in the CCtx. + */ +static void ZSTD_clearAllDicts(ZSTD_CCtx* cctx) +{ + ZSTD_customFree(cctx->localDict.dictBuffer, cctx->customMem); + ZSTD_freeCDict(cctx->localDict.cdict); + ZSTD_memset(&cctx->localDict, 0, sizeof(cctx->localDict)); + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); + cctx->cdict = NULL; +} + +static size_t ZSTD_sizeof_localDict(ZSTD_localDict dict) +{ + size_t const bufferSize = dict.dictBuffer != NULL ? dict.dictSize : 0; + size_t const cdictSize = ZSTD_sizeof_CDict(dict.cdict); + return bufferSize + cdictSize; +} + +static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) +{ + assert(cctx != NULL); + assert(cctx->staticSize == 0); + ZSTD_clearAllDicts(cctx); +#ifdef ZSTD_MULTITHREAD + ZSTDMT_freeCCtx(cctx->mtctx); cctx->mtctx = NULL; +#endif + ZSTD_cwksp_free(&cctx->workspace, cctx->customMem); +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "not compatible with static CCtx"); + { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + ZSTD_freeCCtxContent(cctx); + if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem); + } + return 0; +} + + +static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_sizeof_CCtx(cctx->mtctx); +#else + (void)cctx; + return 0; +#endif +} + + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + /* cctx may be in the workspace */ + return (cctx->workspace.workspace == cctx ? 0 : sizeof(*cctx)) + + ZSTD_cwksp_sizeof(&cctx->workspace) + + ZSTD_sizeof_localDict(cctx->localDict) + + ZSTD_sizeof_mtctx(cctx); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + return ZSTD_sizeof_CCtx(zcs); /* same object */ +} + +/* private API call, for dictBuilder only */ +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); } + +/* Returns true if the strategy supports using a row based matchfinder */ +static int ZSTD_rowMatchFinderSupported(const ZSTD_strategy strategy) { + return (strategy >= ZSTD_greedy && strategy <= ZSTD_lazy2); +} + +/* Returns true if the strategy and useRowMatchFinder mode indicate that we will use the row based matchfinder + * for this compression. + */ +static int ZSTD_rowMatchFinderUsed(const ZSTD_strategy strategy, const ZSTD_paramSwitch_e mode) { + assert(mode != ZSTD_ps_auto); + return ZSTD_rowMatchFinderSupported(strategy) && (mode == ZSTD_ps_enable); +} + +/* Returns row matchfinder usage given an initial mode and cParams */ +static ZSTD_paramSwitch_e ZSTD_resolveRowMatchFinderMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { +#if defined(ZSTD_ARCH_X86_SSE2) || defined(ZSTD_ARCH_ARM_NEON) + int const kHasSIMD128 = 1; +#else + int const kHasSIMD128 = 0; +#endif + if (mode != ZSTD_ps_auto) return mode; /* if requested enabled, but no SIMD, we still will use row matchfinder */ + mode = ZSTD_ps_disable; + if (!ZSTD_rowMatchFinderSupported(cParams->strategy)) return mode; + if (kHasSIMD128) { + if (cParams->windowLog > 14) mode = ZSTD_ps_enable; + } else { + if (cParams->windowLog > 17) mode = ZSTD_ps_enable; + } + return mode; +} + +/* Returns block splitter usage (generally speaking, when using slower/stronger compression modes) */ +static ZSTD_paramSwitch_e ZSTD_resolveBlockSplitterMode(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 17) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +/* Returns 1 if the arguments indicate that we should allocate a chainTable, 0 otherwise */ +static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, + const ZSTD_paramSwitch_e useRowMatchFinder, + const U32 forDDSDict) { + assert(useRowMatchFinder != ZSTD_ps_auto); + /* We always should allocate a chaintable if we are allocating a matchstate for a DDS dictionary matchstate. + * We do not allocate a chaintable if we are using ZSTD_fast, or are using the row-based matchfinder. + */ + return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); +} + +/* Returns ZSTD_ps_enable if compression parameters are such that we should + * enable long distance matching (wlog >= 27, strategy >= btopt). + * Returns ZSTD_ps_disable otherwise. + */ +static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, + const ZSTD_compressionParameters* const cParams) { + if (mode != ZSTD_ps_auto) return mode; + return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable; +} + +static int ZSTD_resolveExternalSequenceValidation(int mode) { + return mode; +} + +/* Resolves maxBlockSize to the default if no value is present. */ +static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { + if (maxBlockSize == 0) { + return ZSTD_BLOCKSIZE_MAX; + } else { + return maxBlockSize; + } +} + +static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) { + if (value != ZSTD_ps_auto) return value; + if (cLevel < 10) { + return ZSTD_ps_disable; + } else { + return ZSTD_ps_enable; + } +} + +/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged. + * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */ +static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) { + return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast; +} + +static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( + ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params cctxParams; + /* should not matter, as all cParams are presumed properly defined */ + ZSTD_CCtxParams_init(&cctxParams, ZSTD_CLEVEL_DEFAULT); + cctxParams.cParams = cParams; + + /* Adjust advanced params according to cParams */ + cctxParams.ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams.ldmParams.enableLdm, &cParams); + if (cctxParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_ldm_adjustParameters(&cctxParams.ldmParams, &cParams); + assert(cctxParams.ldmParams.hashLog >= cctxParams.ldmParams.bucketSizeLog); + assert(cctxParams.ldmParams.hashRateLog < 32); + } + cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams); + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences); + cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize); + cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes, + cctxParams.compressionLevel); + assert(!ZSTD_checkCParams(cParams)); + return cctxParams; +} + +static ZSTD_CCtx_params* ZSTD_createCCtxParams_advanced( + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params* params; + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + params = (ZSTD_CCtx_params*)ZSTD_customCalloc( + sizeof(ZSTD_CCtx_params), customMem); + if (!params) { return NULL; } + ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); + params->customMem = customMem; + return params; +} + +ZSTD_CCtx_params* ZSTD_createCCtxParams(void) +{ + return ZSTD_createCCtxParams_advanced(ZSTD_defaultCMem); +} + +size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params) +{ + if (params == NULL) { return 0; } + ZSTD_customFree(params, params->customMem); + return 0; +} + +size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params) +{ + return ZSTD_CCtxParams_init(params, ZSTD_CLEVEL_DEFAULT); +} + +size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) { + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->compressionLevel = compressionLevel; + cctxParams->fParams.contentSizeFlag = 1; + return 0; +} + +#define ZSTD_NO_CLEVEL 0 + +/** + * Initializes `cctxParams` from `params` and `compressionLevel`. + * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. + */ +static void +ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, + const ZSTD_parameters* params, + int compressionLevel) +{ + assert(!ZSTD_checkCParams(params->cParams)); + ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = compressionLevel; + cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); + cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams); + cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); + cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences); + cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); + cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel); + DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", + cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm); +} + +size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params) +{ + RETURN_ERROR_IF(!cctxParams, GENERIC, "NULL pointer!"); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_init_internal(cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return 0; +} + +/** + * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. + * @param params Validated zstd parameters. + */ +static void ZSTD_CCtxParams_setZstdParams( + ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) +{ + assert(!ZSTD_checkCParams(params->cParams)); + cctxParams->cParams = params->cParams; + cctxParams->fParams = params->fParams; + /* Should not matter, as all cParams are presumed properly defined. + * But, set it for tracing anyway. + */ + cctxParams->compressionLevel = ZSTD_NO_CLEVEL; +} + +ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + + switch(param) + { + case ZSTD_c_compressionLevel: + bounds.lowerBound = ZSTD_minCLevel(); + bounds.upperBound = ZSTD_maxCLevel(); + return bounds; + + case ZSTD_c_windowLog: + bounds.lowerBound = ZSTD_WINDOWLOG_MIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + + case ZSTD_c_hashLog: + bounds.lowerBound = ZSTD_HASHLOG_MIN; + bounds.upperBound = ZSTD_HASHLOG_MAX; + return bounds; + + case ZSTD_c_chainLog: + bounds.lowerBound = ZSTD_CHAINLOG_MIN; + bounds.upperBound = ZSTD_CHAINLOG_MAX; + return bounds; + + case ZSTD_c_searchLog: + bounds.lowerBound = ZSTD_SEARCHLOG_MIN; + bounds.upperBound = ZSTD_SEARCHLOG_MAX; + return bounds; + + case ZSTD_c_minMatch: + bounds.lowerBound = ZSTD_MINMATCH_MIN; + bounds.upperBound = ZSTD_MINMATCH_MAX; + return bounds; + + case ZSTD_c_targetLength: + bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; + bounds.upperBound = ZSTD_TARGETLENGTH_MAX; + return bounds; + + case ZSTD_c_strategy: + bounds.lowerBound = ZSTD_STRATEGY_MIN; + bounds.upperBound = ZSTD_STRATEGY_MAX; + return bounds; + + case ZSTD_c_contentSizeFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_checksumFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_dictIDFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_nbWorkers: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_NBWORKERS_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_jobSize: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_JOBSIZE_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_overlapLog: +#ifdef ZSTD_MULTITHREAD + bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; + bounds.upperBound = ZSTD_OVERLAPLOG_MAX; +#else + bounds.lowerBound = 0; + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_enableDedicatedDictSearch: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_enableLongDistanceMatching: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; + + case ZSTD_c_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; + + case ZSTD_c_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; + + case ZSTD_c_ldmHashRateLog: + bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; + return bounds; + + /* experimental parameters */ + case ZSTD_c_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_forceMaxWindow : + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_format: + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + bounds.lowerBound = ZSTD_f_zstd1; + bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_forceAttachDict: + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceLoad); + bounds.lowerBound = ZSTD_dictDefaultAttach; + bounds.upperBound = ZSTD_dictForceLoad; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_literalCompressionMode: + ZSTD_STATIC_ASSERT(ZSTD_ps_auto < ZSTD_ps_enable && ZSTD_ps_enable < ZSTD_ps_disable); + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_srcSizeHint: + bounds.lowerBound = ZSTD_SRCSIZEHINT_MIN; + bounds.upperBound = ZSTD_SRCSIZEHINT_MAX; + return bounds; + + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + + case ZSTD_c_blockDelimiters: + bounds.lowerBound = (int)ZSTD_sf_noBlockDelimiters; + bounds.upperBound = (int)ZSTD_sf_explicitBlockDelimiters; + return bounds; + + case ZSTD_c_validateSequences: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_useBlockSplitter: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_useRowMatchFinder: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_deterministicRefPrefix: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_prefetchCDictTables: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_enableSeqProducerFallback: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_searchForExternalRepcodes: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + default: + bounds.error = ERROR(parameter_unsupported); + return bounds; + } +} + +/* ZSTD_cParam_clampBounds: + * Clamps the value into the bounded range. + */ +static size_t ZSTD_cParam_clampBounds(ZSTD_cParameter cParam, int* value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return bounds.error; + if (*value < bounds.lowerBound) *value = bounds.lowerBound; + if (*value > bounds.upperBound) *value = bounds.upperBound; + return 0; +} + +#define BOUNDCHECK(cParam, val) { \ + RETURN_ERROR_IF(!ZSTD_cParam_withinBounds(cParam,val), \ + parameter_outOfBound, "Param out of bounds"); \ +} + + +static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) +{ + switch(param) + { + case ZSTD_c_compressionLevel: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + return 1; + + case ZSTD_c_format: + case ZSTD_c_windowLog: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow : + case ZSTD_c_nbWorkers: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_useBlockSplitter: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_searchForExternalRepcodes: + default: + return 0; + } +} + +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); + if (cctx->streamStage != zcss_init) { + if (ZSTD_isUpdateAuthorized(param)) { + cctx->cParamsChanged = 1; + } else { + RETURN_ERROR(stage_wrong, "can only set params in cctx init stage"); + } } + + switch(param) + { + case ZSTD_c_nbWorkers: + RETURN_ERROR_IF((value!=0) && cctx->staticSize, parameter_unsupported, + "MT not compatible with static alloc"); + break; + + case ZSTD_c_compressionLevel: + case ZSTD_c_windowLog: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_format: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow: + case ZSTD_c_forceAttachDict: + case ZSTD_c_literalCompressionMode: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableDedicatedDictSearch: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: + case ZSTD_c_srcSizeHint: + case ZSTD_c_stableInBuffer: + case ZSTD_c_stableOutBuffer: + case ZSTD_c_blockDelimiters: + case ZSTD_c_validateSequences: + case ZSTD_c_useBlockSplitter: + case ZSTD_c_useRowMatchFinder: + case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_searchForExternalRepcodes: + break; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return ZSTD_CCtxParams_setParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, + ZSTD_cParameter param, int value) +{ + DEBUGLOG(4, "ZSTD_CCtxParams_setParameter (%i, %i)", (int)param, value); + switch(param) + { + case ZSTD_c_format : + BOUNDCHECK(ZSTD_c_format, value); + CCtxParams->format = (ZSTD_format_e)value; + return (size_t)CCtxParams->format; + + case ZSTD_c_compressionLevel : { + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + if (value == 0) + CCtxParams->compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else + CCtxParams->compressionLevel = value; + if (CCtxParams->compressionLevel >= 0) return (size_t)CCtxParams->compressionLevel; + return 0; /* return type (size_t) cannot represent negative values */ + } + + case ZSTD_c_windowLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_windowLog, value); + CCtxParams->cParams.windowLog = (U32)value; + return CCtxParams->cParams.windowLog; + + case ZSTD_c_hashLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_hashLog, value); + CCtxParams->cParams.hashLog = (U32)value; + return CCtxParams->cParams.hashLog; + + case ZSTD_c_chainLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_chainLog, value); + CCtxParams->cParams.chainLog = (U32)value; + return CCtxParams->cParams.chainLog; + + case ZSTD_c_searchLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_searchLog, value); + CCtxParams->cParams.searchLog = (U32)value; + return (size_t)value; + + case ZSTD_c_minMatch : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_minMatch, value); + CCtxParams->cParams.minMatch = (U32)value; + return CCtxParams->cParams.minMatch; + + case ZSTD_c_targetLength : + BOUNDCHECK(ZSTD_c_targetLength, value); + CCtxParams->cParams.targetLength = (U32)value; + return CCtxParams->cParams.targetLength; + + case ZSTD_c_strategy : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_strategy, value); + CCtxParams->cParams.strategy = (ZSTD_strategy)value; + return (size_t)CCtxParams->cParams.strategy; + + case ZSTD_c_contentSizeFlag : + /* Content size written in frame header _when known_ (default:1) */ + DEBUGLOG(4, "set content size flag = %u", (value!=0)); + CCtxParams->fParams.contentSizeFlag = value != 0; + return (size_t)CCtxParams->fParams.contentSizeFlag; + + case ZSTD_c_checksumFlag : + /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ + CCtxParams->fParams.checksumFlag = value != 0; + return (size_t)CCtxParams->fParams.checksumFlag; + + case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); + CCtxParams->fParams.noDictIDFlag = !value; + return !CCtxParams->fParams.noDictIDFlag; + + case ZSTD_c_forceMaxWindow : + CCtxParams->forceWindow = (value != 0); + return (size_t)CCtxParams->forceWindow; + + case ZSTD_c_forceAttachDict : { + const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; + BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref); + CCtxParams->attachDictPref = pref; + return CCtxParams->attachDictPref; + } + + case ZSTD_c_literalCompressionMode : { + const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value; + BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm); + CCtxParams->literalCompressionMode = lcm; + return CCtxParams->literalCompressionMode; + } + + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + CCtxParams->nbWorkers = value; + return CCtxParams->nbWorkers; +#endif + + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + /* Adjust to the minimum non-default value. */ + if (value != 0 && value < ZSTDMT_JOBSIZE_MIN) + value = ZSTDMT_JOBSIZE_MIN; + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(param, &value), ""); + assert(value >= 0); + CCtxParams->jobSize = value; + return CCtxParams->jobSize; +#endif + + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->overlapLog = value; + return CCtxParams->overlapLog; +#endif + + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR_IF(value!=0, parameter_unsupported, "not compiled with multithreading"); + return 0; +#else + FORWARD_IF_ERROR(ZSTD_cParam_clampBounds(ZSTD_c_overlapLog, &value), ""); + CCtxParams->rsyncable = value; + return CCtxParams->rsyncable; +#endif + + case ZSTD_c_enableDedicatedDictSearch : + CCtxParams->enableDedicatedDictSearch = (value!=0); + return (size_t)CCtxParams->enableDedicatedDictSearch; + + case ZSTD_c_enableLongDistanceMatching : + BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value); + CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value; + return CCtxParams->ldmParams.enableLdm; + + case ZSTD_c_ldmHashLog : + if (value!=0) /* 0 ==> auto */ + BOUNDCHECK(ZSTD_c_ldmHashLog, value); + CCtxParams->ldmParams.hashLog = (U32)value; + return CCtxParams->ldmParams.hashLog; + + case ZSTD_c_ldmMinMatch : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmMinMatch, value); + CCtxParams->ldmParams.minMatchLength = (U32)value; + return CCtxParams->ldmParams.minMatchLength; + + case ZSTD_c_ldmBucketSizeLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); + CCtxParams->ldmParams.bucketSizeLog = (U32)value; + return CCtxParams->ldmParams.bucketSizeLog; + + case ZSTD_c_ldmHashRateLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmHashRateLog, value); + CCtxParams->ldmParams.hashRateLog = (U32)value; + return CCtxParams->ldmParams.hashRateLog; + + case ZSTD_c_targetCBlockSize : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + CCtxParams->targetCBlockSize = (U32)value; + return CCtxParams->targetCBlockSize; + + case ZSTD_c_srcSizeHint : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_srcSizeHint, value); + CCtxParams->srcSizeHint = value; + return (size_t)CCtxParams->srcSizeHint; + + case ZSTD_c_stableInBuffer: + BOUNDCHECK(ZSTD_c_stableInBuffer, value); + CCtxParams->inBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->inBufferMode; + + case ZSTD_c_stableOutBuffer: + BOUNDCHECK(ZSTD_c_stableOutBuffer, value); + CCtxParams->outBufferMode = (ZSTD_bufferMode_e)value; + return CCtxParams->outBufferMode; + + case ZSTD_c_blockDelimiters: + BOUNDCHECK(ZSTD_c_blockDelimiters, value); + CCtxParams->blockDelimiters = (ZSTD_sequenceFormat_e)value; + return CCtxParams->blockDelimiters; + + case ZSTD_c_validateSequences: + BOUNDCHECK(ZSTD_c_validateSequences, value); + CCtxParams->validateSequences = value; + return CCtxParams->validateSequences; + + case ZSTD_c_useBlockSplitter: + BOUNDCHECK(ZSTD_c_useBlockSplitter, value); + CCtxParams->useBlockSplitter = (ZSTD_paramSwitch_e)value; + return CCtxParams->useBlockSplitter; + + case ZSTD_c_useRowMatchFinder: + BOUNDCHECK(ZSTD_c_useRowMatchFinder, value); + CCtxParams->useRowMatchFinder = (ZSTD_paramSwitch_e)value; + return CCtxParams->useRowMatchFinder; + + case ZSTD_c_deterministicRefPrefix: + BOUNDCHECK(ZSTD_c_deterministicRefPrefix, value); + CCtxParams->deterministicRefPrefix = !!value; + return CCtxParams->deterministicRefPrefix; + + case ZSTD_c_prefetchCDictTables: + BOUNDCHECK(ZSTD_c_prefetchCDictTables, value); + CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value; + return CCtxParams->prefetchCDictTables; + + case ZSTD_c_enableSeqProducerFallback: + BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value); + CCtxParams->enableMatchFinderFallback = value; + return CCtxParams->enableMatchFinderFallback; + + case ZSTD_c_maxBlockSize: + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_maxBlockSize, value); + CCtxParams->maxBlockSize = value; + return CCtxParams->maxBlockSize; + + case ZSTD_c_searchForExternalRepcodes: + BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value); + CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value; + return CCtxParams->searchForExternalRepcodes; + + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } +} + +size_t ZSTD_CCtx_getParameter(ZSTD_CCtx const* cctx, ZSTD_cParameter param, int* value) +{ + return ZSTD_CCtxParams_getParameter(&cctx->requestedParams, param, value); +} + +size_t ZSTD_CCtxParams_getParameter( + ZSTD_CCtx_params const* CCtxParams, ZSTD_cParameter param, int* value) +{ + switch(param) + { + case ZSTD_c_format : + *value = CCtxParams->format; + break; + case ZSTD_c_compressionLevel : + *value = CCtxParams->compressionLevel; + break; + case ZSTD_c_windowLog : + *value = (int)CCtxParams->cParams.windowLog; + break; + case ZSTD_c_hashLog : + *value = (int)CCtxParams->cParams.hashLog; + break; + case ZSTD_c_chainLog : + *value = (int)CCtxParams->cParams.chainLog; + break; + case ZSTD_c_searchLog : + *value = CCtxParams->cParams.searchLog; + break; + case ZSTD_c_minMatch : + *value = CCtxParams->cParams.minMatch; + break; + case ZSTD_c_targetLength : + *value = CCtxParams->cParams.targetLength; + break; + case ZSTD_c_strategy : + *value = (unsigned)CCtxParams->cParams.strategy; + break; + case ZSTD_c_contentSizeFlag : + *value = CCtxParams->fParams.contentSizeFlag; + break; + case ZSTD_c_checksumFlag : + *value = CCtxParams->fParams.checksumFlag; + break; + case ZSTD_c_dictIDFlag : + *value = !CCtxParams->fParams.noDictIDFlag; + break; + case ZSTD_c_forceMaxWindow : + *value = CCtxParams->forceWindow; + break; + case ZSTD_c_forceAttachDict : + *value = CCtxParams->attachDictPref; + break; + case ZSTD_c_literalCompressionMode : + *value = CCtxParams->literalCompressionMode; + break; + case ZSTD_c_nbWorkers : +#ifndef ZSTD_MULTITHREAD + assert(CCtxParams->nbWorkers == 0); +#endif + *value = CCtxParams->nbWorkers; + break; + case ZSTD_c_jobSize : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + assert(CCtxParams->jobSize <= INT_MAX); + *value = (int)CCtxParams->jobSize; + break; +#endif + case ZSTD_c_overlapLog : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->overlapLog; + break; +#endif + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + RETURN_ERROR(parameter_unsupported, "not compiled with multithreading"); +#else + *value = CCtxParams->rsyncable; + break; +#endif + case ZSTD_c_enableDedicatedDictSearch : + *value = CCtxParams->enableDedicatedDictSearch; + break; + case ZSTD_c_enableLongDistanceMatching : + *value = CCtxParams->ldmParams.enableLdm; + break; + case ZSTD_c_ldmHashLog : + *value = CCtxParams->ldmParams.hashLog; + break; + case ZSTD_c_ldmMinMatch : + *value = CCtxParams->ldmParams.minMatchLength; + break; + case ZSTD_c_ldmBucketSizeLog : + *value = CCtxParams->ldmParams.bucketSizeLog; + break; + case ZSTD_c_ldmHashRateLog : + *value = CCtxParams->ldmParams.hashRateLog; + break; + case ZSTD_c_targetCBlockSize : + *value = (int)CCtxParams->targetCBlockSize; + break; + case ZSTD_c_srcSizeHint : + *value = (int)CCtxParams->srcSizeHint; + break; + case ZSTD_c_stableInBuffer : + *value = (int)CCtxParams->inBufferMode; + break; + case ZSTD_c_stableOutBuffer : + *value = (int)CCtxParams->outBufferMode; + break; + case ZSTD_c_blockDelimiters : + *value = (int)CCtxParams->blockDelimiters; + break; + case ZSTD_c_validateSequences : + *value = (int)CCtxParams->validateSequences; + break; + case ZSTD_c_useBlockSplitter : + *value = (int)CCtxParams->useBlockSplitter; + break; + case ZSTD_c_useRowMatchFinder : + *value = (int)CCtxParams->useRowMatchFinder; + break; + case ZSTD_c_deterministicRefPrefix: + *value = (int)CCtxParams->deterministicRefPrefix; + break; + case ZSTD_c_prefetchCDictTables: + *value = (int)CCtxParams->prefetchCDictTables; + break; + case ZSTD_c_enableSeqProducerFallback: + *value = CCtxParams->enableMatchFinderFallback; + break; + case ZSTD_c_maxBlockSize: + *value = (int)CCtxParams->maxBlockSize; + break; + case ZSTD_c_searchForExternalRepcodes: + *value = (int)CCtxParams->searchForExternalRepcodes; + break; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); + } + return 0; +} + +/** ZSTD_CCtx_setParametersUsingCCtxParams() : + * just applies `params` into `cctx` + * no action is performed, parameters are merely stored. + * If ZSTDMT is enabled, parameters are pushed to cctx->mtctx. + * This is possible even if a compression is ongoing. + * In which case, new parameters will be applied on the fly, starting with next compression job. + */ +size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParametersUsingCCtxParams"); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "The context is in the wrong stage!"); + RETURN_ERROR_IF(cctx->cdict, stage_wrong, + "Can't override parameters with cdict attached (some must " + "be inherited from the cdict)."); + + cctx->requestedParams = *params; + return 0; +} + +size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams) +{ + ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setCParams"); + /* only update if all parameters are valid */ + FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), ""); + return 0; +} + +size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams) +{ + ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setFParams"); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), ""); + return 0; +} + +size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParams"); + /* First check cParams, because we want to update all or none. */ + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + /* Next set fParams, because this could fail if the cctx isn't in init stage. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), ""); + /* Finally set cParams, which should succeed. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), ""); + return 0; +} + +size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't set pledgedSrcSize when not in init stage."); + cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; + return 0; +} + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams( + int const compressionLevel, + size_t const dictSize); +static int ZSTD_dedicatedDictSearch_isSupported( + const ZSTD_compressionParameters* cParams); +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams); + +/** + * Initializes the local dictionary using requested parameters. + * NOTE: Initialization does not employ the pledged src size, + * because the dictionary may be used for multiple compressions. + */ +static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) +{ + ZSTD_localDict* const dl = &cctx->localDict; + if (dl->dict == NULL) { + /* No local dictionary. */ + assert(dl->dictBuffer == NULL); + assert(dl->cdict == NULL); + assert(dl->dictSize == 0); + return 0; + } + if (dl->cdict != NULL) { + /* Local dictionary already initialized. */ + assert(cctx->cdict == dl->cdict); + return 0; + } + assert(dl->dictSize > 0); + assert(cctx->cdict == NULL); + assert(cctx->prefixDict.dict == NULL); + + dl->cdict = ZSTD_createCDict_advanced2( + dl->dict, + dl->dictSize, + ZSTD_dlm_byRef, + dl->dictContentType, + &cctx->requestedParams, + cctx->customMem); + RETURN_ERROR_IF(!dl->cdict, memory_allocation, "ZSTD_createCDict_advanced failed"); + cctx->cdict = dl->cdict; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_advanced( + ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when cctx is not in init stage."); + ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */ + if (dict == NULL || dictSize == 0) /* no dictionary */ + return 0; + if (dictLoadMethod == ZSTD_dlm_byRef) { + cctx->localDict.dict = dict; + } else { + /* copy dictionary content inside CCtx to own its lifetime */ + void* dictBuffer; + RETURN_ERROR_IF(cctx->staticSize, memory_allocation, + "static CCtx can't allocate for an internal copy of dictionary"); + dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); + RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation, + "allocation failed for dictionary content"); + ZSTD_memcpy(dictBuffer, dict, dictSize); + cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */ + cctx->localDict.dict = dictBuffer; /* read-only reference */ + } + cctx->localDict.dictSize = dictSize; + cctx->localDict.dictContentType = dictContentType; + return 0; +} + +size_t ZSTD_CCtx_loadDictionary_byReference( + ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + return ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + + +size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a dict when ctx not in init stage."); + /* Free the existing local cdict (if any) to save memory. */ + ZSTD_clearAllDicts(cctx); + cctx->cdict = cdict; + return 0; +} + +size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a pool when ctx not in init stage."); + cctx->pool = pool; + return 0; +} + +size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_CCtx_refPrefix_advanced(cctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + +size_t ZSTD_CCtx_refPrefix_advanced( + ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't ref a prefix when ctx not in init stage."); + ZSTD_clearAllDicts(cctx); + if (prefix != NULL && prefixSize > 0) { + cctx->prefixDict.dict = prefix; + cctx->prefixDict.dictSize = prefixSize; + cctx->prefixDict.dictContentType = dictContentType; + } + return 0; +} + +/*! ZSTD_CCtx_reset() : + * Also dumps dictionary */ +size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Reset parameters is only possible during init stage."); + ZSTD_clearAllDicts(cctx); + ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx)); + return ZSTD_CCtxParams_reset(&cctx->requestedParams); + } + return 0; +} + + +/** ZSTD_checkCParams() : + control CParam values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ + BOUNDCHECK(ZSTD_c_windowLog, (int)cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, (int)cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, (int)cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, (int)cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, (int)cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,(int)cParams.targetLength); + BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); + return 0; +} + +/** ZSTD_clampCParams() : + * make CParam values within valid range. + * @return : valid CParams */ +static ZSTD_compressionParameters +ZSTD_clampCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMP_TYPE(cParam, val, type) { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ + } +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, unsigned) + CLAMP(ZSTD_c_windowLog, cParams.windowLog); + CLAMP(ZSTD_c_chainLog, cParams.chainLog); + CLAMP(ZSTD_c_hashLog, cParams.hashLog); + CLAMP(ZSTD_c_searchLog, cParams.searchLog); + CLAMP(ZSTD_c_minMatch, cParams.minMatch); + CLAMP(ZSTD_c_targetLength,cParams.targetLength); + CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); + return cParams; +} + +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + +/** ZSTD_dictAndWindowLog() : + * Returns an adjusted window log that is large enough to fit the source and the dictionary. + * The zstd format says that the entire dictionary is valid if one byte of the dictionary + * is within the window. So the hashLog and chainLog should be large enough to reference both + * the dictionary and the window. So we must use this adjusted dictAndWindowLog when downsizing + * the hashLog and windowLog. + * NOTE: srcSize must not be ZSTD_CONTENTSIZE_UNKNOWN. + */ +static U32 ZSTD_dictAndWindowLog(U32 windowLog, U64 srcSize, U64 dictSize) +{ + const U64 maxWindowSize = 1ULL << ZSTD_WINDOWLOG_MAX; + /* No dictionary ==> No change */ + if (dictSize == 0) { + return windowLog; + } + assert(windowLog <= ZSTD_WINDOWLOG_MAX); + assert(srcSize != ZSTD_CONTENTSIZE_UNKNOWN); /* Handled in ZSTD_adjustCParams_internal() */ + { + U64 const windowSize = 1ULL << windowLog; + U64 const dictAndWindowSize = dictSize + windowSize; + /* If the window size is already large enough to fit both the source and the dictionary + * then just use the window size. Otherwise adjust so that it fits the dictionary and + * the window. + */ + if (windowSize >= dictSize + srcSize) { + return windowLog; /* Window size large enough already */ + } else if (dictAndWindowSize >= maxWindowSize) { + return ZSTD_WINDOWLOG_MAX; /* Larger than max window log */ + } else { + return ZSTD_highbit32((U32)dictAndWindowSize - 1) + 1; + } + } +} + +/** ZSTD_adjustCParams_internal() : + * optimize `cPar` for a specified input (`srcSize` and `dictSize`). + * mostly downsize to reduce memory consumption and initialization latency. + * `srcSize` can be ZSTD_CONTENTSIZE_UNKNOWN when not known. + * `mode` is the mode for parameter adjustment. See docs for `ZSTD_cParamMode_e`. + * note : `srcSize==0` means 0! + * condition : cPar is presumed validated (can be checked using ZSTD_checkCParams()). */ +static ZSTD_compressionParameters +ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize, + ZSTD_cParamMode_e mode, + ZSTD_paramSwitch_e useRowMatchFinder) +{ + const U64 minSrcSize = 513; /* (1<<9) + 1 */ + const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); + assert(ZSTD_checkCParams(cPar)==0); + + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + /* If we don't know the source size, don't make any + * assumptions about it. We will already have selected + * smaller parameters if a dictionary is in use. + */ + break; + case ZSTD_cpm_createCDict: + /* Assume a small source size when creating a dictionary + * with an unknown source size. + */ + if (dictSize && srcSize == ZSTD_CONTENTSIZE_UNKNOWN) + srcSize = minSrcSize; + break; + case ZSTD_cpm_attachDict: + /* Dictionary has its own dedicated parameters which have + * already been selected. We are selecting parameters + * for only the source. + */ + dictSize = 0; + break; + default: + assert(0); + break; + } + + /* resize windowLog if input is small enough, to use less memory */ + if ( (srcSize <= maxWindowResize) + && (dictSize <= maxWindowResize) ) { + U32 const tSize = (U32)(srcSize + dictSize); + static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; + U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : + ZSTD_highbit32(tSize-1) + 1; + if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; + } + if (srcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const dictAndWindowLog = ZSTD_dictAndWindowLog(cPar.windowLog, (U64)srcSize, (U64)dictSize); + U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cPar.hashLog > dictAndWindowLog+1) cPar.hashLog = dictAndWindowLog+1; + if (cycleLog > dictAndWindowLog) + cPar.chainLog -= (cycleLog - dictAndWindowLog); + } + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) + cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ + + /* We can't use more than 32 bits of hash in total, so that means that we require: + * (hashLog + 8) <= 32 && (chainLog + 8) <= 32 + */ + if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) { + U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; + if (cPar.hashLog > maxShortCacheHashLog) { + cPar.hashLog = maxShortCacheHashLog; + } + if (cPar.chainLog > maxShortCacheHashLog) { + cPar.chainLog = maxShortCacheHashLog; + } + } + + + /* At this point, we aren't 100% sure if we are using the row match finder. + * Unless it is explicitly disabled, conservatively assume that it is enabled. + * In this case it will only be disabled for small sources, so shrinking the + * hash log a little bit shouldn't result in any ratio loss. + */ + if (useRowMatchFinder == ZSTD_ps_auto) + useRowMatchFinder = ZSTD_ps_enable; + + /* We can't hash more than 32-bits in total. So that means that we require: + * (hashLog - rowLog + 8) <= 32 + */ + if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) { + /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cPar.searchLog, 6); + U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; + U32 const maxHashLog = maxRowHashLog + rowLog; + assert(cPar.hashLog >= rowLog); + if (cPar.hashLog > maxHashLog) { + cPar.hashLog = maxHashLog; + } + } + + return cPar; +} + +ZSTD_compressionParameters +ZSTD_adjustCParams(ZSTD_compressionParameters cPar, + unsigned long long srcSize, + size_t dictSize) +{ + cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ + if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto); +} + +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +static void ZSTD_overrideCParams( + ZSTD_compressionParameters* cParams, + const ZSTD_compressionParameters* overrides) +{ + if (overrides->windowLog) cParams->windowLog = overrides->windowLog; + if (overrides->hashLog) cParams->hashLog = overrides->hashLog; + if (overrides->chainLog) cParams->chainLog = overrides->chainLog; + if (overrides->searchLog) cParams->searchLog = overrides->searchLog; + if (overrides->minMatch) cParams->minMatch = overrides->minMatch; + if (overrides->targetLength) cParams->targetLength = overrides->targetLength; + if (overrides->strategy) cParams->strategy = overrides->strategy; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + ZSTD_compressionParameters cParams; + if (srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN && CCtxParams->srcSizeHint > 0) { + srcSizeHint = CCtxParams->srcSizeHint; + } + cParams = ZSTD_getCParams_internal(CCtxParams->compressionLevel, srcSizeHint, dictSize, mode); + if (CCtxParams->ldmParams.enableLdm == ZSTD_ps_enable) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG; + ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); + assert(!ZSTD_checkCParams(cParams)); + /* srcSizeHint == 0 means 0 */ + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder); +} + +static size_t +ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + const ZSTD_paramSwitch_e useRowMatchFinder, + const U32 enableDedicatedDictSearch, + const U32 forCCtx) +{ + /* chain table size should be 0 for fast or row-hash strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, enableDedicatedDictSearch && !forCCtx) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + /* We don't use ZSTD_cwksp_alloc_size() here because the tables aren't + * surrounded by redzones in ASAN. */ + size_t const tableSpace = chainSize * sizeof(U32) + + hSize * sizeof(U32) + + h3Size * sizeof(U32); + size_t const optPotentialSpace = + ZSTD_cwksp_aligned_alloc_size((MaxML+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((MaxLL+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((MaxOff+1) * sizeof(U32)) + + ZSTD_cwksp_aligned_alloc_size((1<strategy, useRowMatchFinder) + ? ZSTD_cwksp_aligned_alloc_size(hSize) + : 0; + size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) + ? optPotentialSpace + : 0; + size_t const slackSpace = ZSTD_cwksp_slack_space_required(); + + /* tables are guaranteed to be sized in multiples of 64 bytes (or 16 uint32_t) */ + ZSTD_STATIC_ASSERT(ZSTD_HASHLOG_MIN >= 4 && ZSTD_WINDOWLOG_MIN >= 4 && ZSTD_CHAINLOG_MIN >= 4); + assert(useRowMatchFinder != ZSTD_ps_auto); + + DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", + (U32)chainSize, (U32)hSize, (U32)h3Size); + return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; +} + +/* Helper function for calculating memory requirements. + * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ +static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { + U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4; + return blockSize / divider; +} + +static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( + const ZSTD_compressionParameters* cParams, + const ldmParams_t* ldmParams, + const int isStatic, + const ZSTD_paramSwitch_e useRowMatchFinder, + const size_t buffInSize, + const size_t buffOutSize, + const U64 pledgedSrcSize, + int useSequenceProducer, + size_t maxBlockSize) +{ + size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); + size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) + + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); + size_t const entropySpace = ZSTD_cwksp_alloc_size(ENTROPY_WORKSPACE_SIZE); + size_t const blockStateSpace = 2 * ZSTD_cwksp_alloc_size(sizeof(ZSTD_compressedBlockState_t)); + size_t const matchStateSize = ZSTD_sizeof_matchState(cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 0, /* forCCtx */ 1); + + size_t const ldmSpace = ZSTD_ldm_getTableSize(*ldmParams); + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(*ldmParams, blockSize); + size_t const ldmSeqSpace = ldmParams->enableLdm == ZSTD_ps_enable ? + ZSTD_cwksp_aligned_alloc_size(maxNbLdmSeq * sizeof(rawSeq)) : 0; + + + size_t const bufferSpace = ZSTD_cwksp_alloc_size(buffInSize) + + ZSTD_cwksp_alloc_size(buffOutSize); + + size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + size_t const externalSeqSpace = useSequenceProducer + ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) + : 0; + + size_t const neededSpace = + cctxSpace + + entropySpace + + blockStateSpace + + ldmSpace + + ldmSeqSpace + + matchStateSize + + tokenSpace + + bufferSpace + + externalSeqSpace; + + DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); + return neededSpace; +} + +size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, + &cParams); + + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + /* estimateCCtxSize is for one-shot compression. So no buffers should + * be needed. However, we still allocate two 0-sized buffers, which can + * take space under ASAN. */ + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize); +} + +size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCCtxSize_usingCCtxParams(&initialParams); + } +} + +static size_t ZSTD_estimateCCtxSize_internal(int compressionLevel) +{ + int tier = 0; + size_t largestSize = 0; + static const unsigned long long srcSizeTiers[4] = {16 KB, 128 KB, 256 KB, ZSTD_CONTENTSIZE_UNKNOWN}; + for (; tier < 4; ++tier) { + /* Choose the set of cParams for a given level across all srcSizes that give the largest cctxSize */ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeTiers[tier], 0, ZSTD_cpm_noAttachDict); + largestSize = MAX(ZSTD_estimateCCtxSize_usingCParams(cParams), largestSize); + } + return largestSize; +} + +size_t ZSTD_estimateCCtxSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + /* Ensure monotonically increasing memory usage as compression level increases */ + size_t const newMB = ZSTD_estimateCCtxSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) +{ + RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); + { ZSTD_compressionParameters const cParams = + ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog); + size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) + ? ((size_t)1 << cParams.windowLog) + blockSize + : 0; + size_t const outBuffSize = (params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params->useRowMatchFinder, ¶ms->cParams); + + return ZSTD_estimateCCtxSize_usingCCtxParams_internal( + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, + ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize); + } +} + +size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams) +{ + ZSTD_CCtx_params initialParams = ZSTD_makeCCtxParamsFromCParams(cParams); + if (ZSTD_rowMatchFinderSupported(cParams.strategy)) { + /* Pick bigger of not using and using row-based matchfinder for greedy and lazy strategies */ + size_t noRowCCtxSize; + size_t rowCCtxSize; + initialParams.useRowMatchFinder = ZSTD_ps_disable; + noRowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + initialParams.useRowMatchFinder = ZSTD_ps_enable; + rowCCtxSize = ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + return MAX(noRowCCtxSize, rowCCtxSize); + } else { + return ZSTD_estimateCStreamSize_usingCCtxParams(&initialParams); + } +} + +static size_t ZSTD_estimateCStreamSize_internal(int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + return ZSTD_estimateCStreamSize_usingCParams(cParams); +} + +size_t ZSTD_estimateCStreamSize(int compressionLevel) +{ + int level; + size_t memBudget = 0; + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { + size_t const newMB = ZSTD_estimateCStreamSize_internal(level); + if (newMB > memBudget) memBudget = newMB; + } + return memBudget; +} + +/* ZSTD_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads (non-blocking mode). + */ +ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_getFrameProgression(cctx->mtctx); + } +#endif + { ZSTD_frameProgression fp; + size_t const buffered = (cctx->inBuff == NULL) ? 0 : + cctx->inBuffPos - cctx->inToCompress; + if (buffered) assert(cctx->inBuffPos >= cctx->inToCompress); + assert(buffered <= ZSTD_BLOCKSIZE_MAX); + fp.ingested = cctx->consumedSrcSize + buffered; + fp.consumed = cctx->consumedSrcSize; + fp.produced = cctx->producedCSize; + fp.flushed = cctx->producedCSize; /* simplified; some data might still be left within streaming output buffer */ + fp.currentJobID = 0; + fp.nbActiveWorkers = 0; + return fp; +} } + +/*! ZSTD_toFlushNow() + * Only useful for multithreading scenarios currently (nbWorkers >= 1). + */ +size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + return ZSTDMT_toFlushNow(cctx->mtctx); + } +#endif + (void)cctx; + return 0; /* over-simplification; could also check if context is currently running in streaming mode, and in which case, report how many bytes are left to be flushed within output buffer */ +} + +static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, + ZSTD_compressionParameters cParams2) +{ + (void)cParams1; + (void)cParams2; + assert(cParams1.windowLog == cParams2.windowLog); + assert(cParams1.chainLog == cParams2.chainLog); + assert(cParams1.hashLog == cParams2.hashLog); + assert(cParams1.searchLog == cParams2.searchLog); + assert(cParams1.minMatch == cParams2.minMatch); + assert(cParams1.targetLength == cParams2.targetLength); + assert(cParams1.strategy == cParams2.strategy); +} + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs) +{ + int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + bs->rep[i] = repStartValue[i]; + bs->entropy.huf.repeatMode = HUF_repeat_none; + bs->entropy.fse.offcode_repeatMode = FSE_repeat_none; + bs->entropy.fse.matchlength_repeatMode = FSE_repeat_none; + bs->entropy.fse.litlength_repeatMode = FSE_repeat_none; +} + +/*! ZSTD_invalidateMatchState() + * Invalidate all the matches in the match finder tables. + * Requires nextSrc and base to be set (can be NULL). + */ +static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) +{ + ZSTD_window_clear(&ms->window); + + ms->nextToUpdate = ms->window.dictLimit; + ms->loadedDictEnd = 0; + ms->opt.litLengthSum = 0; /* force reset of btopt stats */ + ms->dictMatchState = NULL; +} + +/** + * Controls, for this matchState reset, whether the tables need to be cleared / + * prepared for the coming compression (ZSTDcrp_makeClean), or whether the + * tables can be left unclean (ZSTDcrp_leaveDirty), because we know that a + * subsequent operation will overwrite the table space anyways (e.g., copying + * the matchState contents in from a CDict). + */ +typedef enum { + ZSTDcrp_makeClean, + ZSTDcrp_leaveDirty +} ZSTD_compResetPolicy_e; + +/** + * Controls, for this matchState reset, whether indexing can continue where it + * left off (ZSTDirp_continue), or whether it needs to be restarted from zero + * (ZSTDirp_reset). + */ +typedef enum { + ZSTDirp_continue, + ZSTDirp_reset +} ZSTD_indexResetPolicy_e; + +typedef enum { + ZSTD_resetTarget_CDict, + ZSTD_resetTarget_CCtx +} ZSTD_resetTarget_e; + +/* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */ +static U64 ZSTD_bitmix(U64 val, U64 len) { + val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24); + val *= 0x9FB21C651E98DF25ULL; + val ^= (val >> 35) + len ; + val *= 0x9FB21C651E98DF25ULL; + return val ^ (val >> 28); +} + +/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */ +static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) { + ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4); +} + +static size_t +ZSTD_reset_matchState(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + const ZSTD_compressionParameters* cParams, + const ZSTD_paramSwitch_e useRowMatchFinder, + const ZSTD_compResetPolicy_e crp, + const ZSTD_indexResetPolicy_e forceResetIndex, + const ZSTD_resetTarget_e forWho) +{ + /* disable chain table allocation for fast or row-based strategies */ + size_t const chainSize = ZSTD_allocateChainTable(cParams->strategy, useRowMatchFinder, + ms->dedicatedDictSearch && (forWho == ZSTD_resetTarget_CDict)) + ? ((size_t)1 << cParams->chainLog) + : 0; + size_t const hSize = ((size_t)1) << cParams->hashLog; + U32 const hashLog3 = ((forWho == ZSTD_resetTarget_CCtx) && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + size_t const h3Size = hashLog3 ? ((size_t)1) << hashLog3 : 0; + + DEBUGLOG(4, "reset indices : %u", forceResetIndex == ZSTDirp_reset); + assert(useRowMatchFinder != ZSTD_ps_auto); + if (forceResetIndex == ZSTDirp_reset) { + ZSTD_window_init(&ms->window); + ZSTD_cwksp_mark_tables_dirty(ws); + } + + ms->hashLog3 = hashLog3; + ms->lazySkipping = 0; + + ZSTD_invalidateMatchState(ms); + + assert(!ZSTD_cwksp_reserve_failed(ws)); /* check that allocation hasn't already failed */ + + ZSTD_cwksp_clear_tables(ws); + + DEBUGLOG(5, "reserving table space"); + /* table Space */ + ms->hashTable = (U32*)ZSTD_cwksp_reserve_table(ws, hSize * sizeof(U32)); + ms->chainTable = (U32*)ZSTD_cwksp_reserve_table(ws, chainSize * sizeof(U32)); + ms->hashTable3 = (U32*)ZSTD_cwksp_reserve_table(ws, h3Size * sizeof(U32)); + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + + DEBUGLOG(4, "reset table : %u", crp!=ZSTDcrp_leaveDirty); + if (crp!=ZSTDcrp_leaveDirty) { + /* reset tables only */ + ZSTD_cwksp_clean_tables(ws); + } + + if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { + /* Row match finder needs an additional table of hashes ("tags") */ + size_t const tagTableSize = hSize; + /* We want to generate a new salt in case we reset a Cctx, but we always want to use + * 0 when we reset a Cdict */ + if(forWho == ZSTD_resetTarget_CCtx) { + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); + ZSTD_advanceHashSalt(ms); + } else { + /* When we are not salting we want to always memset the memory */ + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ms->hashSalt = 0; + } + { /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); + assert(cParams->hashLog >= rowLog); + ms->rowHashLog = cParams->hashLog - rowLog; + } + } + + /* opt parser space */ + if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { + DEBUGLOG(4, "reserving optimal parser space"); + ms->opt.litFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (1<opt.litLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxLL+1) * sizeof(unsigned)); + ms->opt.matchLengthFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxML+1) * sizeof(unsigned)); + ms->opt.offCodeFreq = (unsigned*)ZSTD_cwksp_reserve_aligned(ws, (MaxOff+1) * sizeof(unsigned)); + ms->opt.matchTable = (ZSTD_match_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)); + ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); + } + + ms->cParams = *cParams; + + RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, + "failed a workspace allocation in ZSTD_reset_matchState"); + return 0; +} + +/* ZSTD_indexTooCloseToMax() : + * minor optimization : prefer memset() rather than reduceIndex() + * which is measurably slow in some circumstances (reported for Visual Studio). + * Works when re-using a context for a lot of smallish inputs : + * if all inputs are smaller than ZSTD_INDEXOVERFLOW_MARGIN, + * memset() will be triggered before reduceIndex(). + */ +#define ZSTD_INDEXOVERFLOW_MARGIN (16 MB) +static int ZSTD_indexTooCloseToMax(ZSTD_window_t w) +{ + return (size_t)(w.nextSrc - w.base) > (ZSTD_CURRENT_MAX - ZSTD_INDEXOVERFLOW_MARGIN); +} + +/** ZSTD_dictTooBig(): + * When dictionaries are larger than ZSTD_CHUNKSIZE_MAX they can't be loaded in + * one go generically. So we ensure that in that case we reset the tables to zero, + * so that we can load as much of the dictionary as possible. + */ +static int ZSTD_dictTooBig(size_t const loadedDictSize) +{ + return loadedDictSize > ZSTD_CHUNKSIZE_MAX; +} + +/*! ZSTD_resetCCtx_internal() : + * @param loadedDictSize The size of the dictionary to be loaded + * into the context, if any. If no dictionary is used, or the + * dictionary is being attached / copied, then pass 0. + * note : `params` are assumed fully validated at this stage. + */ +static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, + ZSTD_CCtx_params const* params, + U64 const pledgedSrcSize, + size_t const loadedDictSize, + ZSTD_compResetPolicy_e const crp, + ZSTD_buffered_policy_e const zbuff) +{ + ZSTD_cwksp* const ws = &zc->workspace; + DEBUGLOG(4, "ZSTD_resetCCtx_internal: pledgedSrcSize=%u, wlog=%u, useRowMatchFinder=%d useBlockSplitter=%d", + (U32)pledgedSrcSize, params->cParams.windowLog, (int)params->useRowMatchFinder, (int)params->useBlockSplitter); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + + zc->isFirstBlock = 1; + + /* Set applied params early so we can modify them for LDM, + * and point params at the applied params. + */ + zc->appliedParams = *params; + params = &zc->appliedParams; + + assert(params->useRowMatchFinder != ZSTD_ps_auto); + assert(params->useBlockSplitter != ZSTD_ps_auto); + assert(params->ldmParams.enableLdm != ZSTD_ps_auto); + assert(params->maxBlockSize != 0); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* Adjust long distance matching parameters */ + ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); + assert(params->ldmParams.hashLog >= params->ldmParams.bucketSizeLog); + assert(params->ldmParams.hashRateLog < 32); + } + + { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); + size_t const blockSize = MIN(params->maxBlockSize, windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useSequenceProducer); + size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) + ? ZSTD_compressBound(blockSize) + 1 + : 0; + size_t const buffInSize = (zbuff == ZSTDb_buffered && params->inBufferMode == ZSTD_bm_buffered) + ? windowSize + blockSize + : 0; + size_t const maxNbLdmSeq = ZSTD_ldm_getMaxNbSeq(params->ldmParams, blockSize); + + int const indexTooClose = ZSTD_indexTooCloseToMax(zc->blockState.matchState.window); + int const dictTooBig = ZSTD_dictTooBig(loadedDictSize); + ZSTD_indexResetPolicy_e needsIndexReset = + (indexTooClose || dictTooBig || !zc->initialized) ? ZSTDirp_reset : ZSTDirp_continue; + + size_t const neededSpace = + ZSTD_estimateCCtxSize_usingCCtxParams_internal( + ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, + buffInSize, buffOutSize, pledgedSrcSize, params->useSequenceProducer, params->maxBlockSize); + int resizeWorkspace; + + FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); + + if (!zc->staticSize) ZSTD_cwksp_bump_oversized_duration(ws, 0); + + { /* Check if workspace is large enough, alloc a new one if needed */ + int const workspaceTooSmall = ZSTD_cwksp_sizeof(ws) < neededSpace; + int const workspaceWasteful = ZSTD_cwksp_check_wasteful(ws, neededSpace); + resizeWorkspace = workspaceTooSmall || workspaceWasteful; + DEBUGLOG(4, "Need %zu B workspace", neededSpace); + DEBUGLOG(4, "windowSize: %zu - blockSize: %zu", windowSize, blockSize); + + if (resizeWorkspace) { + DEBUGLOG(4, "Resize workspaceSize from %zuKB to %zuKB", + ZSTD_cwksp_sizeof(ws) >> 10, + neededSpace >> 10); + + RETURN_ERROR_IF(zc->staticSize, memory_allocation, "static cctx : no resize"); + + needsIndexReset = ZSTDirp_reset; + + ZSTD_cwksp_free(ws, zc->customMem); + FORWARD_IF_ERROR(ZSTD_cwksp_create(ws, neededSpace, zc->customMem), ""); + + DEBUGLOG(5, "reserving object space"); + /* Statically sized space. + * entropyWorkspace never moves, + * though prev/next block swap places */ + assert(ZSTD_cwksp_check_available(ws, 2 * sizeof(ZSTD_compressedBlockState_t))); + zc->blockState.prevCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.prevCBlock == NULL, memory_allocation, "couldn't allocate prevCBlock"); + zc->blockState.nextCBlock = (ZSTD_compressedBlockState_t*) ZSTD_cwksp_reserve_object(ws, sizeof(ZSTD_compressedBlockState_t)); + RETURN_ERROR_IF(zc->blockState.nextCBlock == NULL, memory_allocation, "couldn't allocate nextCBlock"); + zc->entropyWorkspace = (U32*) ZSTD_cwksp_reserve_object(ws, ENTROPY_WORKSPACE_SIZE); + RETURN_ERROR_IF(zc->entropyWorkspace == NULL, memory_allocation, "couldn't allocate entropyWorkspace"); + } } + + ZSTD_cwksp_clear(ws); + + /* init params */ + zc->blockState.matchState.cParams = params->cParams; + zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable; + zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; + zc->consumedSrcSize = 0; + zc->producedCSize = 0; + if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) + zc->appliedParams.fParams.contentSizeFlag = 0; + DEBUGLOG(4, "pledged content size : %u ; flag : %u", + (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + zc->blockSize = blockSize; + + XXH64_reset(&zc->xxhState, 0); + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->dictContentSize = 0; + + ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms->cParams, + params->useRowMatchFinder, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx), ""); + + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + + /* ldm hash table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + ZSTD_window_init(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; + } + + /* reserve space for block-level external sequences */ + if (params->useSequenceProducer) { + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq; + zc->externalMatchCtx.seqBuffer = + (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); + } + + /* buffers */ + + /* ZSTD_wildcopy() is used to copy into the literals buffer, + * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. + */ + zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); + zc->seqStore.maxNbLit = blockSize; + + zc->bufferedPolicy = zbuff; + zc->inBuffSize = buffInSize; + zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); + zc->outBuffSize = buffOutSize; + zc->outBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffOutSize); + + /* ldm bucketOffsets table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const numBuckets = + ((size_t)1) << (params->ldmParams.hashLog - + params->ldmParams.bucketSizeLog); + zc->ldmState.bucketOffsets = ZSTD_cwksp_reserve_buffer(ws, numBuckets); + ZSTD_memset(zc->ldmState.bucketOffsets, 0, numBuckets); + } + + /* sequences storage */ + ZSTD_referenceExternalSequences(zc, NULL, 0); + zc->seqStore.maxNbSeq = maxNbSeq; + zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); + + DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); + assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace)); + + zc->initialized = 1; + + return 0; + } +} + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { + int i; + for (i=0; iblockState.prevCBlock->rep[i] = 0; + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); +} + +/* These are the approximate sizes for each strategy past which copying the + * dictionary tables into the working context is faster than using them + * in-place. + */ +static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { + 8 KB, /* unused */ + 8 KB, /* ZSTD_fast */ + 16 KB, /* ZSTD_dfast */ + 32 KB, /* ZSTD_greedy */ + 32 KB, /* ZSTD_lazy */ + 32 KB, /* ZSTD_lazy2 */ + 32 KB, /* ZSTD_btlazy2 */ + 32 KB, /* ZSTD_btopt */ + 8 KB, /* ZSTD_btultra */ + 8 KB /* ZSTD_btultra2 */ +}; + +static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize) +{ + size_t cutoff = attachDictSizeCutoffs[cdict->matchState.cParams.strategy]; + int const dedicatedDictSearch = cdict->matchState.dedicatedDictSearch; + return dedicatedDictSearch + || ( ( pledgedSrcSize <= cutoff + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || params->attachDictPref == ZSTD_dictForceAttach ) + && params->attachDictPref != ZSTD_dictForceCopy + && !params->forceWindow ); /* dictMatchState isn't correctly + * handled in _enforceMaxDist */ +} + +static size_t +ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + DEBUGLOG(4, "ZSTD_resetCCtx_byAttachingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + { + ZSTD_compressionParameters adjusted_cdict_cParams = cdict->matchState.cParams; + unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Resize working context table params for input only, since the dict + * has its own tables. */ + /* pledgedSrcSize == 0 means 0! */ + + if (cdict->matchState.dedicatedDictSearch) { + ZSTD_dedicatedDictSearch_revertCParams(&adjusted_cdict_cParams); + } + + params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, + cdict->dictContentSize, ZSTD_cpm_attachDict, + params.useRowMatchFinder); + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_makeClean, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == adjusted_cdict_cParams.strategy); + } + + { const U32 cdictEnd = (U32)( cdict->matchState.window.nextSrc + - cdict->matchState.window.base); + const U32 cdictLen = cdictEnd - cdict->matchState.window.dictLimit; + if (cdictLen == 0) { + /* don't even attach dictionaries with no contents */ + DEBUGLOG(4, "skipping attaching empty dictionary"); + } else { + DEBUGLOG(4, "attaching dictionary into context"); + cctx->blockState.matchState.dictMatchState = &cdict->matchState; + + /* prep working match state so dict matches never have negative indices + * when they are translated to the working context's index space. */ + if (cctx->blockState.matchState.window.dictLimit < cdictEnd) { + cctx->blockState.matchState.window.nextSrc = + cctx->blockState.matchState.window.base + cdictEnd; + ZSTD_window_clear(&cctx->blockState.matchState.window); + } + /* loadedDictEnd is expressed within the referential of the active context */ + cctx->blockState.matchState.loadedDictEnd = cctx->blockState.matchState.window.dictLimit; + } } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, + ZSTD_compressionParameters const* cParams) { + if (ZSTD_CDictIndicesAreTagged(cParams)){ + /* Remove tags from the CDict table if they are present. + * See docs on "short cache" in zstd_compress_internal.h for context. */ + size_t i; + for (i = 0; i < tableSize; i++) { + U32 const taggedIndex = src[i]; + U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS; + dst[i] = index; + } + } else { + ZSTD_memcpy(dst, src, tableSize * sizeof(U32)); + } +} + +static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + const ZSTD_compressionParameters *cdict_cParams = &cdict->matchState.cParams; + + assert(!cdict->matchState.dedicatedDictSearch); + DEBUGLOG(4, "ZSTD_resetCCtx_byCopyingCDict() pledgedSrcSize=%llu", + (unsigned long long)pledgedSrcSize); + + { unsigned const windowLog = params.cParams.windowLog; + assert(windowLog != 0); + /* Copy only compression parameters related to tables. */ + params.cParams = *cdict_cParams; + params.cParams.windowLog = windowLog; + params.useRowMatchFinder = cdict->useRowMatchFinder; + FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff), ""); + assert(cctx->appliedParams.cParams.strategy == cdict_cParams->strategy); + assert(cctx->appliedParams.cParams.hashLog == cdict_cParams->hashLog); + assert(cctx->appliedParams.cParams.chainLog == cdict_cParams->chainLog); + } + + ZSTD_cwksp_mark_tables_dirty(&cctx->workspace); + assert(params.useRowMatchFinder != ZSTD_ps_auto); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(cdict_cParams->strategy, cdict->useRowMatchFinder, 0 /* DDS guaranteed disabled */) + ? ((size_t)1 << cdict_cParams->chainLog) + : 0; + size_t const hSize = (size_t)1 << cdict_cParams->hashLog; + + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize, cdict_cParams); + + /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ + if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize, cdict_cParams); + } + /* copy tag table */ + if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { + size_t const tagTableSize = hSize; + ZSTD_memcpy(cctx->blockState.matchState.tagTable, + cdict->matchState.tagTable, + tagTableSize); + cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; + } + } + + /* Zero the hashTable3, since the cdict never fills it */ + { int const h3log = cctx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + assert(cdict->matchState.hashLog3 == 0); + ZSTD_memset(cctx->blockState.matchState.hashTable3, 0, h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&cctx->workspace); + + /* copy dictionary offsets */ + { ZSTD_matchState_t const* srcMatchState = &cdict->matchState; + ZSTD_matchState_t* dstMatchState = &cctx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + + cctx->dictID = cdict->dictID; + cctx->dictContentSize = cdict->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(cctx->blockState.prevCBlock, &cdict->cBlockState, sizeof(cdict->cBlockState)); + + return 0; +} + +/* We have a choice between copying the dictionary context into the working + * context, or referencing the dictionary context from the working context + * in-place. We decide here which strategy to use. */ +static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + + DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", + (unsigned)pledgedSrcSize); + + if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { + return ZSTD_resetCCtx_byAttachingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } else { + return ZSTD_resetCCtx_byCopyingCDict( + cctx, cdict, *params, pledgedSrcSize, zbuff); + } +} + +/*! ZSTD_copyCCtx_internal() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * The "context", in this case, refers to the hash and chain tables, + * entropy tables, and dictionary references. + * `windowLog` value is enforced if != 0, otherwise value is copied from srcCCtx. + * @return : 0, or an error code */ +static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, + const ZSTD_CCtx* srcCCtx, + ZSTD_frameParameters fParams, + U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + RETURN_ERROR_IF(srcCCtx->stage!=ZSTDcs_init, stage_wrong, + "Can't copy a ctx that's not in init stage."); + DEBUGLOG(5, "ZSTD_copyCCtx_internal"); + ZSTD_memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + { ZSTD_CCtx_params params = dstCCtx->requestedParams; + /* Copy only compression parameters related to tables. */ + params.cParams = srcCCtx->appliedParams.cParams; + assert(srcCCtx->appliedParams.useRowMatchFinder != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.useBlockSplitter != ZSTD_ps_auto); + assert(srcCCtx->appliedParams.ldmParams.enableLdm != ZSTD_ps_auto); + params.useRowMatchFinder = srcCCtx->appliedParams.useRowMatchFinder; + params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter; + params.ldmParams = srcCCtx->appliedParams.ldmParams; + params.fParams = fParams; + params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; + ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, + /* loadedDictSize */ 0, + ZSTDcrp_leaveDirty, zbuff); + assert(dstCCtx->appliedParams.cParams.windowLog == srcCCtx->appliedParams.cParams.windowLog); + assert(dstCCtx->appliedParams.cParams.strategy == srcCCtx->appliedParams.cParams.strategy); + assert(dstCCtx->appliedParams.cParams.hashLog == srcCCtx->appliedParams.cParams.hashLog); + assert(dstCCtx->appliedParams.cParams.chainLog == srcCCtx->appliedParams.cParams.chainLog); + assert(dstCCtx->blockState.matchState.hashLog3 == srcCCtx->blockState.matchState.hashLog3); + } + + ZSTD_cwksp_mark_tables_dirty(&dstCCtx->workspace); + + /* copy tables */ + { size_t const chainSize = ZSTD_allocateChainTable(srcCCtx->appliedParams.cParams.strategy, + srcCCtx->appliedParams.useRowMatchFinder, + 0 /* forDDSDict */) + ? ((size_t)1 << srcCCtx->appliedParams.cParams.chainLog) + : 0; + size_t const hSize = (size_t)1 << srcCCtx->appliedParams.cParams.hashLog; + int const h3log = srcCCtx->blockState.matchState.hashLog3; + size_t const h3Size = h3log ? ((size_t)1 << h3log) : 0; + + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable, + srcCCtx->blockState.matchState.hashTable, + hSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.chainTable, + srcCCtx->blockState.matchState.chainTable, + chainSize * sizeof(U32)); + ZSTD_memcpy(dstCCtx->blockState.matchState.hashTable3, + srcCCtx->blockState.matchState.hashTable3, + h3Size * sizeof(U32)); + } + + ZSTD_cwksp_mark_tables_clean(&dstCCtx->workspace); + + /* copy dictionary offsets */ + { + const ZSTD_matchState_t* srcMatchState = &srcCCtx->blockState.matchState; + ZSTD_matchState_t* dstMatchState = &dstCCtx->blockState.matchState; + dstMatchState->window = srcMatchState->window; + dstMatchState->nextToUpdate = srcMatchState->nextToUpdate; + dstMatchState->loadedDictEnd= srcMatchState->loadedDictEnd; + } + dstCCtx->dictID = srcCCtx->dictID; + dstCCtx->dictContentSize = srcCCtx->dictContentSize; + + /* copy block state */ + ZSTD_memcpy(dstCCtx->blockState.prevCBlock, srcCCtx->blockState.prevCBlock, sizeof(*srcCCtx->blockState.prevCBlock)); + + return 0; +} + +/*! ZSTD_copyCCtx() : + * Duplicate an existing context `srcCCtx` into another one `dstCCtx`. + * Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). + * pledgedSrcSize==0 means "unknown". +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + ZSTD_frameParameters fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + ZSTD_buffered_policy_e const zbuff = srcCCtx->bufferedPolicy; + ZSTD_STATIC_ASSERT((U32)ZSTDb_buffered==1); + if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; + fParams.contentSizeFlag = (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN); + + return ZSTD_copyCCtx_internal(dstCCtx, srcCCtx, + fParams, pledgedSrcSize, + zbuff); +} + + +#define ZSTD_ROWSIZE 16 +/*! ZSTD_reduceTable() : + * reduce table indexes by `reducerValue`, or squash to zero. + * PreserveMark preserves "unsorted mark" for btlazy2 strategy. + * It must be set to a clear 0/1 value, to remove branch during inlining. + * Presume table size is a multiple of ZSTD_ROWSIZE + * to help auto-vectorization */ +FORCE_INLINE_TEMPLATE void +ZSTD_reduceTable_internal (U32* const table, U32 const size, U32 const reducerValue, int const preserveMark) +{ + int const nbRows = (int)size / ZSTD_ROWSIZE; + int cellNb = 0; + int rowNb; + /* Protect special index values < ZSTD_WINDOW_START_INDEX. */ + U32 const reducerThreshold = reducerValue + ZSTD_WINDOW_START_INDEX; + assert((size & (ZSTD_ROWSIZE-1)) == 0); /* multiple of ZSTD_ROWSIZE */ + assert(size < (1U<<31)); /* can be casted to int */ + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * + * This function however is intended to operate on those dirty tables and + * re-clean them. So when this function is used correctly, we can unpoison + * the memory it operated on. This introduces a blind spot though, since + * if we now try to operate on __actually__ poisoned memory, we will not + * detect that. */ + __msan_unpoison(table, size * sizeof(U32)); +#endif + + for (rowNb=0 ; rowNb < nbRows ; rowNb++) { + int column; + for (column=0; columncParams.hashLog; + ZSTD_reduceTable(ms->hashTable, hSize, reducerValue); + } + + if (ZSTD_allocateChainTable(params->cParams.strategy, params->useRowMatchFinder, (U32)ms->dedicatedDictSearch)) { + U32 const chainSize = (U32)1 << params->cParams.chainLog; + if (params->cParams.strategy == ZSTD_btlazy2) + ZSTD_reduceTable_btlazy2(ms->chainTable, chainSize, reducerValue); + else + ZSTD_reduceTable(ms->chainTable, chainSize, reducerValue); + } + + if (ms->hashLog3) { + U32 const h3Size = (U32)1 << ms->hashLog3; + ZSTD_reduceTable(ms->hashTable3, h3Size, reducerValue); + } +} + + +/*-******************************************************* +* Block entropic compression +*********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + int longOffsets = 0; + assert(nbSeq <= seqStorePtr->maxNbSeq); + for (u=0; u= STREAM_ACCUMULATOR_MIN)); + if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) + longOffsets = 1; + } + if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + return longOffsets; +} + +/* ZSTD_useTargetCBlockSize(): + * Returns if target compressed block size param is being used. + * If used, compression will do best effort to make a compressed block size to be around targetCBlockSize. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_useTargetCBlockSize(const ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_useTargetCBlockSize (targetCBlockSize=%zu)", cctxParams->targetCBlockSize); + return (cctxParams->targetCBlockSize != 0); +} + +/* ZSTD_blockSplitterEnabled(): + * Returns if block splitting param is being used + * If used, compression will do best effort to split a block in order to improve compression ratio. + * At the time this function is called, the parameter must be finalized. + * Returns 1 if true, 0 otherwise. */ +static int ZSTD_blockSplitterEnabled(ZSTD_CCtx_params* cctxParams) +{ + DEBUGLOG(5, "ZSTD_blockSplitterEnabled (useBlockSplitter=%d)", cctxParams->useBlockSplitter); + assert(cctxParams->useBlockSplitter != ZSTD_ps_auto); + return (cctxParams->useBlockSplitter == ZSTD_ps_enable); +} + +/* Type returned by ZSTD_buildSequencesStatistics containing finalized symbol encoding types + * and size of the sequences statistics + */ +typedef struct { + U32 LLtype; + U32 Offtype; + U32 MLtype; + size_t size; + size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ + int longOffsets; +} ZSTD_symbolEncodingTypeStats_t; + +/* ZSTD_buildSequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t, or a zstd error code in the `size` field. + * Modifies `nextEntropy` to have the appropriate values as a side effect. + * nbSeq must be greater than 0. + * + * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildSequencesStatistics( + const seqStore_t* seqStorePtr, size_t nbSeq, + const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, + BYTE* dst, const BYTE* const dstEnd, + ZSTD_strategy strategy, unsigned* countWorkspace, + void* entropyWorkspace, size_t entropyWkspSize) +{ + BYTE* const ostart = dst; + const BYTE* const oend = dstEnd; + BYTE* op = ostart; + FSE_CTable* CTable_LitLength = nextEntropy->litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->matchlengthCTable; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + ZSTD_symbolEncodingTypeStats_t stats; + + stats.lastCountSize = 0; + /* convert length/distances into codes */ + stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); + assert(op <= oend); + assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ + /* build CTable for Literal Lengths */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(countWorkspace, &max, llCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building LL table"); + nextEntropy->litlength_repeatMode = prevEntropy->litlength_repeatMode; + stats.LLtype = ZSTD_selectEncodingType(&nextEntropy->litlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(set_basic < set_compressed && set_rle < set_compressed); + assert(!(stats.LLtype < set_compressed && nextEntropy->litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_LitLength, LLFSELog, (symbolEncodingType_e)stats.LLtype, + countWorkspace, max, llCodeTable, nbSeq, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + prevEntropy->litlengthCTable, + sizeof(prevEntropy->litlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for LitLens failed"); + stats.size = countSize; + return stats; + } + if (stats.LLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for Offsets */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, ofCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ + ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; + DEBUGLOG(5, "Building OF table"); + nextEntropy->offcode_repeatMode = prevEntropy->offcode_repeatMode; + stats.Offtype = ZSTD_selectEncodingType(&nextEntropy->offcode_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); + assert(!(stats.Offtype < set_compressed && nextEntropy->offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)stats.Offtype, + countWorkspace, max, ofCodeTable, nbSeq, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + prevEntropy->offcodeCTable, + sizeof(prevEntropy->offcodeCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for Offsets failed"); + stats.size = countSize; + return stats; + } + if (stats.Offtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + /* build CTable for MatchLengths */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp( + countWorkspace, &max, mlCodeTable, nbSeq, entropyWorkspace, entropyWkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); + nextEntropy->matchlength_repeatMode = prevEntropy->matchlength_repeatMode; + stats.MLtype = ZSTD_selectEncodingType(&nextEntropy->matchlength_repeatMode, + countWorkspace, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); + assert(!(stats.MLtype < set_compressed && nextEntropy->matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ + { size_t const countSize = ZSTD_buildCTable( + op, (size_t)(oend - op), + CTable_MatchLength, MLFSELog, (symbolEncodingType_e)stats.MLtype, + countWorkspace, max, mlCodeTable, nbSeq, + ML_defaultNorm, ML_defaultNormLog, MaxML, + prevEntropy->matchlengthCTable, + sizeof(prevEntropy->matchlengthCTable), + entropyWorkspace, entropyWkspSize); + if (ZSTD_isError(countSize)) { + DEBUGLOG(3, "ZSTD_buildCTable for MatchLengths failed"); + stats.size = countSize; + return stats; + } + if (stats.MLtype == set_compressed) + stats.lastCountSize = countSize; + op += countSize; + assert(op <= oend); + } } + stats.size = (size_t)(op-ostart); + return stats; +} + +/* ZSTD_entropyCompressSeqStore_internal(): + * compresses both literals and sequences + * Returns compressed size of block, or a zstd error. + */ +#define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore_internal( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + void* entropyWorkspace, size_t entropyWkspSize, + const int bmi2) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + unsigned* count = (unsigned*)entropyWorkspace; + FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; + FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; + FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; + const seqDef* const sequences = seqStorePtr->sequencesStart; + const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t lastCountSize; + int longOffsets = 0; + + entropyWorkspace = count + (MaxSeq + 1); + entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); + + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity); + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); + + /* Compress literals */ + { const BYTE* const literals = seqStorePtr->litStart; + size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + /* Base suspicion of uncompressibility on ratio of literals to sequences */ + unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); + size_t const litSize = (size_t)(seqStorePtr->lit - literals); + + size_t const cSize = ZSTD_compressLiterals( + op, dstCapacity, + literals, litSize, + entropyWorkspace, entropyWkspSize, + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + suspectUncompressible, bmi2); + FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); + assert(cSize <= dstCapacity); + op += cSize; + } + + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, "Can't fit seq hdr in output buf!"); + if (nbSeq < 128) { + *op++ = (BYTE)nbSeq; + } else if (nbSeq < LONGNBSEQ) { + op[0] = (BYTE)((nbSeq>>8) + 0x80); + op[1] = (BYTE)nbSeq; + op+=2; + } else { + op[0]=0xFF; + MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)); + op+=3; + } + assert(op <= oend); + if (nbSeq==0) { + /* Copy the old tables over as if we repeated them */ + ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); + return (size_t)(op - ostart); + } + { BYTE* const seqHead = op++; + /* build stats for sequences */ + const ZSTD_symbolEncodingTypeStats_t stats = + ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + &prevEntropy->fse, &nextEntropy->fse, + op, oend, + strategy, count, + entropyWorkspace, entropyWkspSize); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); + lastCountSize = stats.lastCountSize; + op += stats.size; + longOffsets = stats.longOffsets; + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, (size_t)(oend - op), + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + assert(op <= oend); + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ + if (lastCountSize && (lastCountSize + bitstreamSize) < 4) { + /* lastCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } + } + + DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); + return (size_t)(op - ostart); +} + +MEM_STATIC size_t +ZSTD_entropyCompressSeqStore( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) +{ + size_t const cSize = ZSTD_entropyCompressSeqStore_internal( + seqStorePtr, prevEntropy, nextEntropy, cctxParams, + dst, dstCapacity, + entropyWorkspace, entropyWkspSize, bmi2); + if (cSize == 0) return 0; + /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. + * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. + */ + if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) { + DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity); + return 0; /* block not compressed */ + } + FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); + + /* Check compressibility */ + { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); + if (cSize >= maxCSize) return 0; /* block not compressed */ + } + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); + /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly. + * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above. + */ + assert(cSize < ZSTD_BLOCKSIZE_MAX); + return cSize; +} + +/* ZSTD_selectBlockCompressor() : + * Not static, but internal use only (used by long distance matcher) + * assumption : strat is a valid strategy */ +ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_paramSwitch_e useRowMatchFinder, ZSTD_dictMode_e dictMode) +{ + static const ZSTD_blockCompressor blockCompressor[4][ZSTD_STRATEGY_MAX+1] = { + { ZSTD_compressBlock_fast /* default for 0 */, + ZSTD_compressBlock_fast, + ZSTD_compressBlock_doubleFast, + ZSTD_compressBlock_greedy, + ZSTD_compressBlock_lazy, + ZSTD_compressBlock_lazy2, + ZSTD_compressBlock_btlazy2, + ZSTD_compressBlock_btopt, + ZSTD_compressBlock_btultra, + ZSTD_compressBlock_btultra2 }, + { ZSTD_compressBlock_fast_extDict /* default for 0 */, + ZSTD_compressBlock_fast_extDict, + ZSTD_compressBlock_doubleFast_extDict, + ZSTD_compressBlock_greedy_extDict, + ZSTD_compressBlock_lazy_extDict, + ZSTD_compressBlock_lazy2_extDict, + ZSTD_compressBlock_btlazy2_extDict, + ZSTD_compressBlock_btopt_extDict, + ZSTD_compressBlock_btultra_extDict, + ZSTD_compressBlock_btultra_extDict }, + { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, + ZSTD_compressBlock_fast_dictMatchState, + ZSTD_compressBlock_doubleFast_dictMatchState, + ZSTD_compressBlock_greedy_dictMatchState, + ZSTD_compressBlock_lazy_dictMatchState, + ZSTD_compressBlock_lazy2_dictMatchState, + ZSTD_compressBlock_btlazy2_dictMatchState, + ZSTD_compressBlock_btopt_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState }, + { NULL /* default for 0 */, + NULL, + NULL, + ZSTD_compressBlock_greedy_dedicatedDictSearch, + ZSTD_compressBlock_lazy_dedicatedDictSearch, + ZSTD_compressBlock_lazy2_dedicatedDictSearch, + NULL, + NULL, + NULL, + NULL } + }; + ZSTD_blockCompressor selectedCompressor; + ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); + + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + DEBUGLOG(4, "Selected block compressor: dictMode=%d strat=%d rowMatchfinder=%d", (int)dictMode, (int)strat, (int)useRowMatchFinder); + if (ZSTD_rowMatchFinderUsed(strat, useRowMatchFinder)) { + static const ZSTD_blockCompressor rowBasedBlockCompressors[4][3] = { + { ZSTD_compressBlock_greedy_row, + ZSTD_compressBlock_lazy_row, + ZSTD_compressBlock_lazy2_row }, + { ZSTD_compressBlock_greedy_extDict_row, + ZSTD_compressBlock_lazy_extDict_row, + ZSTD_compressBlock_lazy2_extDict_row }, + { ZSTD_compressBlock_greedy_dictMatchState_row, + ZSTD_compressBlock_lazy_dictMatchState_row, + ZSTD_compressBlock_lazy2_dictMatchState_row }, + { ZSTD_compressBlock_greedy_dedicatedDictSearch_row, + ZSTD_compressBlock_lazy_dedicatedDictSearch_row, + ZSTD_compressBlock_lazy2_dedicatedDictSearch_row } + }; + DEBUGLOG(4, "Selecting a row-based matchfinder"); + assert(useRowMatchFinder != ZSTD_ps_auto); + selectedCompressor = rowBasedBlockCompressors[(int)dictMode][(int)strat - (int)ZSTD_greedy]; + } else { + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; + } + assert(selectedCompressor != NULL); + return selectedCompressor; +} + +static void ZSTD_storeLastLiterals(seqStore_t* seqStorePtr, + const BYTE* anchor, size_t lastLLSize) +{ + ZSTD_memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; +} + +void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthType = ZSTD_llt_none; +} + +/* ZSTD_postProcessSequenceProducerResult() : + * Validates and post-processes sequences obtained through the external matchfinder API: + * - Checks whether nbExternalSeqs represents an error condition. + * - Appends a block delimiter to outSeqs if one is not already present. + * See zstd.h for context regarding block delimiters. + * Returns the number of sequences after post-processing, or an error code. */ +static size_t ZSTD_postProcessSequenceProducerResult( + ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize +) { + RETURN_ERROR_IF( + nbExternalSeqs > outSeqsCapacity, + sequenceProducer_failed, + "External sequence producer returned error code %lu", + (unsigned long)nbExternalSeqs + ); + + RETURN_ERROR_IF( + nbExternalSeqs == 0 && srcSize > 0, + sequenceProducer_failed, + "Got zero sequences from external sequence producer for a non-empty src buffer!" + ); + + if (srcSize == 0) { + ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence)); + return 1; + } + + { + ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1]; + + /* We can return early if lastSeq is already a block delimiter. */ + if (lastSeq.offset == 0 && lastSeq.matchLength == 0) { + return nbExternalSeqs; + } + + /* This error condition is only possible if the external matchfinder + * produced an invalid parse, by definition of ZSTD_sequenceBound(). */ + RETURN_ERROR_IF( + nbExternalSeqs == outSeqsCapacity, + sequenceProducer_failed, + "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!" + ); + + /* lastSeq is not a block delimiter, so we need to append one. */ + ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence)); + return nbExternalSeqs + 1; + } +} + +/* ZSTD_fastSequenceLengthSum() : + * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*. + * Similar to another function in zstd_compress.c (determine_blockSize), + * except it doesn't check for a block delimiter to end summation. + * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P). + * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */ +static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) { + size_t matchLenSum, litLenSum, i; + matchLenSum = 0; + litLenSum = 0; + for (i = 0; i < seqBufSize; i++) { + litLenSum += seqBuf[i].litLength; + matchLenSum += seqBuf[i].matchLength; + } + return litLenSum + matchLenSum; +} + +typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; + +static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + ZSTD_matchState_t* const ms = &zc->blockState.matchState; + DEBUGLOG(5, "ZSTD_buildSeqStore (srcSize=%zu)", srcSize); + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + /* Assert that we have correctly flushed the ctx params into the ms's copy */ + ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { + ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); + } else { + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); + } + return ZSTDbss_noCompress; /* don't even attempt compression below a certain srcSize */ + } + ZSTD_resetSeqStore(&(zc->seqStore)); + /* required for optimal parser to read stats from dictionary */ + ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; + /* tell the optimal parser how we expect to compress literals */ + ms->opt.literalCompressionMode = zc->appliedParams.literalCompressionMode; + /* a gap between an attached dict and the current window is not safe, + * they must remain adjacent, + * and when that stops being the case, the dict must be unset */ + assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); + + /* limited update after a very long match */ + { const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 curr = (U32)(istart-base); + if (sizeof(ptrdiff_t)==8) assert(istart - base < (ptrdiff_t)(U32)(-1)); /* ensure no overflow */ + if (curr > ms->nextToUpdate + 384) + ms->nextToUpdate = curr - MIN(192, (U32)(curr - ms->nextToUpdate - 384)); + } + + /* select and store sequences */ + { ZSTD_dictMode_e const dictMode = ZSTD_matchState_dictMode(ms); + size_t lastLLSize; + { int i; + for (i = 0; i < ZSTD_REP_NUM; ++i) + zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i]; + } + if (zc->externSeqStore.pos < zc->externSeqStore.size) { + assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + zc->appliedParams.useSequenceProducer, + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&zc->externSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(zc->externSeqStore.pos <= zc->externSeqStore.size); + } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + rawSeqStore_t ldmSeqStore = kNullRawSeqStore; + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + zc->appliedParams.useSequenceProducer, + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + + ldmSeqStore.seq = zc->ldmSequences; + ldmSeqStore.capacity = zc->maxNbLdmSequences; + /* Updates ldmSeqStore.size */ + FORWARD_IF_ERROR(ZSTD_ldm_generateSequences(&zc->ldmState, &ldmSeqStore, + &zc->appliedParams.ldmParams, + src, srcSize), ""); + /* Updates ldmSeqStore.pos */ + lastLLSize = + ZSTD_ldm_blockCompress(&ldmSeqStore, + ms, &zc->seqStore, + zc->blockState.nextCBlock->rep, + zc->appliedParams.useRowMatchFinder, + src, srcSize); + assert(ldmSeqStore.pos == ldmSeqStore.size); + } else if (zc->appliedParams.useSequenceProducer) { + assert( + zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize) + ); + assert(zc->externalMatchCtx.mFinder != NULL); + + { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; + + size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)( + zc->externalMatchCtx.mState, + zc->externalMatchCtx.seqBuffer, + zc->externalMatchCtx.seqBufferCapacity, + src, srcSize, + NULL, 0, /* dict and dictSize, currently not supported */ + zc->appliedParams.compressionLevel, + windowSize + ); + + size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( + zc->externalMatchCtx.seqBuffer, + nbExternalSeqs, + zc->externalMatchCtx.seqBufferCapacity, + srcSize + ); + + /* Return early if there is no error, since we don't need to worry about last literals */ + if (!ZSTD_isError(nbPostProcessedSeqs)) { + ZSTD_sequencePosition seqPos = {0,0,0}; + size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs); + RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!"); + FORWARD_IF_ERROR( + ZSTD_copySequencesToSeqStoreExplicitBlockDelim( + zc, &seqPos, + zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs, + src, srcSize, + zc->appliedParams.searchForExternalRepcodes + ), + "Failed to copy external sequences to seqStore!" + ); + ms->ldmSeqStore = NULL; + DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs); + return ZSTDbss_compress; + } + + /* Propagate the error if fallback is disabled */ + if (!zc->appliedParams.enableMatchFinderFallback) { + return nbPostProcessedSeqs; + } + + /* Fallback to software matchfinder */ + { ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + DEBUGLOG( + 5, + "External sequence producer returned error code %lu. Falling back to internal parser.", + (unsigned long)nbExternalSeqs + ); + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } } + } else { /* not long range mode and no external matchfinder */ + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } + { const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize; + ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize); + } } + return ZSTDbss_compress; +} + +static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) +{ + const seqStore_t* seqStore = ZSTD_getSeqStore(zc); + const seqDef* seqStoreSeqs = seqStore->sequencesStart; + size_t seqStoreSeqSize = seqStore->sequences - seqStoreSeqs; + size_t seqStoreLiteralsSize = (size_t)(seqStore->lit - seqStore->litStart); + size_t literalsRead = 0; + size_t lastLLSize; + + ZSTD_Sequence* outSeqs = &zc->seqCollector.seqStart[zc->seqCollector.seqIndex]; + size_t i; + repcodes_t updatedRepcodes; + + assert(zc->seqCollector.seqIndex + 1 < zc->seqCollector.maxSequences); + /* Ensure we have enough space for last literals "sequence" */ + assert(zc->seqCollector.maxSequences >= seqStoreSeqSize + 1); + ZSTD_memcpy(updatedRepcodes.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (i = 0; i < seqStoreSeqSize; ++i) { + U32 rawOffset = seqStoreSeqs[i].offBase - ZSTD_REP_NUM; + outSeqs[i].litLength = seqStoreSeqs[i].litLength; + outSeqs[i].matchLength = seqStoreSeqs[i].mlBase + MINMATCH; + outSeqs[i].rep = 0; + + if (i == seqStore->longLengthPos) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + outSeqs[i].litLength += 0x10000; + } else if (seqStore->longLengthType == ZSTD_llt_matchLength) { + outSeqs[i].matchLength += 0x10000; + } + } + + if (seqStoreSeqs[i].offBase <= ZSTD_REP_NUM) { + /* Derive the correct offset corresponding to a repcode */ + outSeqs[i].rep = seqStoreSeqs[i].offBase; + if (outSeqs[i].litLength != 0) { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep - 1]; + } else { + if (outSeqs[i].rep == 3) { + rawOffset = updatedRepcodes.rep[0] - 1; + } else { + rawOffset = updatedRepcodes.rep[outSeqs[i].rep]; + } + } + } + outSeqs[i].offset = rawOffset; + /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode + so we provide seqStoreSeqs[i].offset - 1 */ + ZSTD_updateRep(updatedRepcodes.rep, + seqStoreSeqs[i].offBase, + seqStoreSeqs[i].litLength == 0); + literalsRead += outSeqs[i].litLength; + } + /* Insert last literals (if any exist) in the block as a sequence with ml == off == 0. + * If there are no last literals, then we'll emit (of: 0, ml: 0, ll: 0), which is a marker + * for the block boundary, according to the API. + */ + assert(seqStoreLiteralsSize >= literalsRead); + lastLLSize = seqStoreLiteralsSize - literalsRead; + outSeqs[i].litLength = (U32)lastLLSize; + outSeqs[i].matchLength = outSeqs[i].offset = outSeqs[i].rep = 0; + seqStoreSeqSize++; + zc->seqCollector.seqIndex += seqStoreSeqSize; +} + +size_t ZSTD_sequenceBound(size_t srcSize) { + return (srcSize / ZSTD_MINMATCH_MIN) + 1; +} + +size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, + size_t outSeqsSize, const void* src, size_t srcSize) +{ + const size_t dstCapacity = ZSTD_compressBound(srcSize); + void* dst = ZSTD_customMalloc(dstCapacity, ZSTD_defaultCMem); + SeqCollector seqCollector; + + RETURN_ERROR_IF(dst == NULL, memory_allocation, "NULL pointer!"); + + seqCollector.collectSequences = 1; + seqCollector.seqStart = outSeqs; + seqCollector.seqIndex = 0; + seqCollector.maxSequences = outSeqsSize; + zc->seqCollector = seqCollector; + + ZSTD_compress2(zc, dst, dstCapacity, src, srcSize); + ZSTD_customFree(dst, ZSTD_defaultCMem); + return zc->seqCollector.seqIndex; +} + +size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize) { + size_t in = 0; + size_t out = 0; + for (; in < seqsSize; ++in) { + if (sequences[in].offset == 0 && sequences[in].matchLength == 0) { + if (in != seqsSize - 1) { + sequences[in+1].litLength += sequences[in].litLength; + } + } else { + sequences[out] = sequences[in]; + ++out; + } + } + return out; +} + +/* Unrolled loop to read four size_ts of input at a time. Returns 1 if is RLE, 0 if not. */ +static int ZSTD_isRLE(const BYTE* src, size_t length) { + const BYTE* ip = src; + const BYTE value = ip[0]; + const size_t valueST = (size_t)((U64)value * 0x0101010101010101ULL); + const size_t unrollSize = sizeof(size_t) * 4; + const size_t unrollMask = unrollSize - 1; + const size_t prefixLength = length & unrollMask; + size_t i; + if (length == 1) return 1; + /* Check if prefix is RLE first before using unrolled loop */ + if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { + return 0; + } + for (i = prefixLength; i != length; i += unrollSize) { + size_t u; + for (u = 0; u < unrollSize; u += sizeof(size_t)) { + if (MEM_readST(ip + i + u) != valueST) { + return 0; + } } } + return 1; +} + +/* Returns true if the given block may be RLE. + * This is just a heuristic based on the compressibility. + * It may return both false positives and false negatives. + */ +static int ZSTD_maybeRLE(seqStore_t const* seqStore) +{ + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t const nbLits = (size_t)(seqStore->lit - seqStore->litStart); + + return nbSeqs < 4 && nbLits < 10; +} + +static void +ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) +{ + ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; + bs->prevCBlock = bs->nextCBlock; + bs->nextCBlock = tmp; +} + +/* Writes the block header */ +static void +writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) +{ + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + DEBUGLOG(3, "writeBlockHeader: cSize: %zu blockSize: %zu lastBlock: %u", cSize, blockSize, lastBlock); +} + +/** ZSTD_buildBlockEntropyStats_literals() : + * Builds entropy for the literals. + * Stores literals block type (raw, rle, compressed, repeat) and + * huffman description table to hufMetadata. + * Requires ENTROPY_WORKSPACE_SIZE workspace + * @return : size of huffman description table, or an error code + */ +static size_t +ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int literalsCompressionIsDisabled, + void* workspace, size_t wkspSize, + int hufFlags) +{ + BYTE* const wkspStart = (BYTE*)workspace; + BYTE* const wkspEnd = wkspStart + wkspSize; + BYTE* const countWkspStart = wkspStart; + unsigned* const countWksp = (unsigned*)workspace; + const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); + BYTE* const nodeWksp = countWkspStart + countWkspSize; + const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + unsigned huffLog = LitHufLog; + HUF_repeat repeat = prevHuf->repeatMode; + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (literalsCompressionIsDisabled) { + DEBUGLOG(5, "set_basic - disabled"); + hufMetadata->hType = set_basic; + return 0; + } + + /* small ? don't even attempt compression (speed opt) */ +#ifndef COMPRESS_LITERALS_SIZE_MIN +# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */ +#endif + { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; + if (srcSize <= minLitSize) { + DEBUGLOG(5, "set_basic - too small"); + hufMetadata->hType = set_basic; + return 0; + } } + + /* Scan input and build symbol stats */ + { size_t const largest = + HIST_count_wksp (countWksp, &maxSymbolValue, + (const BYTE*)src, srcSize, + workspace, wkspSize); + FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); + if (largest == srcSize) { + /* only one literal symbol */ + DEBUGLOG(5, "set_rle"); + hufMetadata->hType = set_rle; + return 0; + } + if (largest <= (srcSize >> 7)+4) { + /* heuristic: likely not compressible */ + DEBUGLOG(5, "set_basic - no gain"); + hufMetadata->hType = set_basic; + return 0; + } } + + /* Validate the previous Huffman table */ + if (repeat == HUF_repeat_check + && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + repeat = HUF_repeat_none; + } + + /* Build Huffman Tree */ + ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags); + assert(huffLog <= LitHufLog); + { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, + maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); + huffLog = (U32)maxBits; + } + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable_wksp( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_repeat; + return 0; + } } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; + } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; + } +} + + +/* ZSTD_buildDummySequencesStatistics(): + * Returns a ZSTD_symbolEncodingTypeStats_t with all encoding types as set_basic, + * and updates nextEntropy to the appropriate repeatMode. + */ +static ZSTD_symbolEncodingTypeStats_t +ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) +{ + ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; + nextEntropy->litlength_repeatMode = FSE_repeat_none; + nextEntropy->offcode_repeatMode = FSE_repeat_none; + nextEntropy->matchlength_repeatMode = FSE_repeat_none; + return stats; +} + +/** ZSTD_buildBlockEntropyStats_sequences() : + * Builds entropy for the sequences. + * Stores symbol compression modes and fse table to fseMetadata. + * Requires ENTROPY_WORKSPACE_SIZE wksp. + * @return : size of fse tables or error code */ +static size_t +ZSTD_buildBlockEntropyStats_sequences( + const seqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) +{ + ZSTD_strategy const strategy = cctxParams->cParams.strategy; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + BYTE* const ostart = fseMetadata->fseTablesBuffer; + BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); + BYTE* op = ostart; + unsigned* countWorkspace = (unsigned*)workspace; + unsigned* entropyWorkspace = countWorkspace + (MaxSeq + 1); + size_t entropyWorkspaceSize = wkspSize - (MaxSeq + 1) * sizeof(*countWorkspace); + ZSTD_symbolEncodingTypeStats_t stats; + + DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_sequences (nbSeq=%zu)", nbSeq); + stats = nbSeq != 0 ? ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + prevEntropy, nextEntropy, op, oend, + strategy, countWorkspace, + entropyWorkspace, entropyWorkspaceSize) + : ZSTD_buildDummySequencesStatistics(nextEntropy); + FORWARD_IF_ERROR(stats.size, "ZSTD_buildSequencesStatistics failed!"); + fseMetadata->llType = (symbolEncodingType_e) stats.LLtype; + fseMetadata->ofType = (symbolEncodingType_e) stats.Offtype; + fseMetadata->mlType = (symbolEncodingType_e) stats.MLtype; + fseMetadata->lastCountSize = stats.lastCountSize; + return stats.size; +} + + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * Requires workspace size ENTROPY_WORKSPACE_SIZE + * @return : 0 on success, or an error code + * Note : also employed in superblock + */ +size_t ZSTD_buildBlockEntropyStats( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) +{ + size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); + int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0; + + entropyMetadata->hufMetadata.hufDesSize = + ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, + &prevEntropy->huf, &nextEntropy->huf, + &entropyMetadata->hufMetadata, + ZSTD_literalsCompressionIsDisabled(cctxParams), + workspace, wkspSize, hufFlags); + + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); + entropyMetadata->fseMetadata.fseTablesSize = + ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, + &prevEntropy->fse, &nextEntropy->fse, + cctxParams, + &entropyMetadata->fseMetadata, + workspace, wkspSize); + FORWARD_IF_ERROR(entropyMetadata->fseMetadata.fseTablesSize, "ZSTD_buildBlockEntropyStats_sequences failed"); + return 0; +} + +/* Returns the size estimate for the literals section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; + size_t literalSectionHeaderSize = 3 + (litSize >= 1 KB) + (litSize >= 16 KB); + U32 singleStream = litSize < 256; + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + if (!singleStream) cLitSizeEstimate += 6; /* multi-stream huffman uses 6-byte jump table */ + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +/* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ +static size_t +ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, size_t nbSeq, unsigned maxCode, + const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + (void)defaultMax; + cSymbolTypeSizeEstimateInBits = ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) { + return nbSeq * 10; + } + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits >> 3; +} + +/* Returns the size estimate for the sequences section (header + content) of a block */ +static size_t +ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); + size_t cSeqSizeEstimate = 0; + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, + fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, + fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, + fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +/* Returns the size estimate for a given stream of literals, of, ll, ml */ +static size_t +ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) +{ + size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return seqSize + literalsSize + ZSTD_blockHeaderSize; +} + +/* Builds entropy statistics and uses them for blocksize estimation. + * + * @return: estimated compressed size of the seqStore, or a zstd error. + */ +static size_t +ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) +{ + ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; + DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), ""); + return ZSTD_estimateBlockSize( + seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), + seqStore->ofCode, seqStore->llCode, seqStore->mlCode, + (size_t)(seqStore->sequences - seqStore->sequencesStart), + &zc->blockState.nextCBlock->entropy, + entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, + (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); +} + +/* Returns literals bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) +{ + size_t literalsBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + seqDef const seq = seqStore->sequencesStart[i]; + literalsBytes += seq.litLength; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { + literalsBytes += 0x10000; + } } + return literalsBytes; +} + +/* Returns match bytes represented in a seqStore */ +static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) +{ + size_t matchBytes = 0; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); + size_t i; + for (i = 0; i < nbSeqs; ++i) { + seqDef seq = seqStore->sequencesStart[i]; + matchBytes += seq.mlBase + MINMATCH; + if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { + matchBytes += 0x10000; + } } + return matchBytes; +} + +/* Derives the seqStore that is a chunk of the originalSeqStore from [startIdx, endIdx). + * Stores the result in resultSeqStore. + */ +static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, + const seqStore_t* originalSeqStore, + size_t startIdx, size_t endIdx) +{ + *resultSeqStore = *originalSeqStore; + if (startIdx > 0) { + resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + } + + /* Move longLengthPos into the correct position if necessary */ + if (originalSeqStore->longLengthType != ZSTD_llt_none) { + if (originalSeqStore->longLengthPos < startIdx || originalSeqStore->longLengthPos > endIdx) { + resultSeqStore->longLengthType = ZSTD_llt_none; + } else { + resultSeqStore->longLengthPos -= (U32)startIdx; + } + } + resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; + resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; + if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { + /* This accounts for possible last literals if the derived chunk reaches the end of the block */ + assert(resultSeqStore->lit == originalSeqStore->lit); + } else { + size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + resultSeqStore->lit = resultSeqStore->litStart + literalsBytes; + } + resultSeqStore->llCode += startIdx; + resultSeqStore->mlCode += startIdx; + resultSeqStore->ofCode += startIdx; +} + +/** + * Returns the raw offset represented by the combination of offBase, ll0, and repcode history. + * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq(). + */ +static U32 +ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0) +{ + U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */ + assert(OFFBASE_IS_REPCODE(offBase)); + if (adjustedRepCode == ZSTD_REP_NUM) { + assert(ll0); + /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 + * This is only valid if it results in a valid offset value, aka > 0. + * Note : it may happen that `rep[0]==1` in exceptional circumstances. + * In which case this function will return 0, which is an invalid offset. + * It's not an issue though, since this value will be + * compared and discarded within ZSTD_seqStore_resolveOffCodes(). + */ + return rep[0] - 1; + } + return rep[adjustedRepCode]; +} + +/** + * ZSTD_seqStore_resolveOffCodes() reconciles any possible divergences in offset history that may arise + * due to emission of RLE/raw blocks that disturb the offset history, + * and replaces any repcodes within the seqStore that may be invalid. + * + * dRepcodes are updated as would be on the decompression side. + * cRepcodes are updated exactly in accordance with the seqStore. + * + * Note : this function assumes seq->offBase respects the following numbering scheme : + * 0 : invalid + * 1-3 : repcode 1-3 + * 4+ : real_offset+3 + */ +static void +ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, + const seqStore_t* const seqStore, U32 const nbSeq) +{ + U32 idx = 0; + U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq; + for (; idx < nbSeq; ++idx) { + seqDef* const seq = seqStore->sequencesStart + idx; + U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx); + U32 const offBase = seq->offBase; + assert(offBase > 0); + if (OFFBASE_IS_REPCODE(offBase)) { + U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0); + U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0); + /* Adjust simulated decompression repcode history if we come across a mismatch. Replace + * the repcode with the offset it actually references, determined by the compression + * repcode history. + */ + if (dRawOffset != cRawOffset) { + seq->offBase = OFFSET_TO_OFFBASE(cRawOffset); + } + } + /* Compression repcode history is always updated with values directly from the unmodified seqStore. + * Decompression repcode history may use modified seq->offset value taken from compression repcode history. + */ + ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0); + ZSTD_updateRep(cRepcodes->rep, offBase, ll0); + } +} + +/* ZSTD_compressSeqStore_singleBlock(): + * Compresses a seqStore into a block with a block header, into the buffer dst. + * + * Returns the total size of that block (including header) or a ZSTD error code. + */ +static size_t +ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, + const seqStore_t* const seqStore, + repcodes_t* const dRep, repcodes_t* const cRep, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock, U32 isPartition) +{ + const U32 rleMaxLength = 25; + BYTE* op = (BYTE*)dst; + const BYTE* ip = (const BYTE*)src; + size_t cSize; + size_t cSeqsSize; + + /* In case of an RLE or raw block, the simulated decompression repcode history must be reset */ + repcodes_t const dRepOriginal = *dRep; + DEBUGLOG(5, "ZSTD_compressSeqStore_singleBlock"); + if (isPartition) + ZSTD_seqStore_resolveOffCodes(dRep, cRep, seqStore, (U32)(seqStore->sequences - seqStore->sequencesStart)); + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "Block header doesn't fit"); + cSeqsSize = ZSTD_entropyCompressSeqStore(seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + op + ZSTD_blockHeaderSize, dstCapacity - ZSTD_blockHeaderSize, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + FORWARD_IF_ERROR(cSeqsSize, "ZSTD_entropyCompressSeqStore failed!"); + + if (!zc->isFirstBlock && + cSeqsSize < rleMaxLength && + ZSTD_isRLE((BYTE const*)src, srcSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + cSeqsSize = 1; + } + + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + if (cSeqsSize == 0) { + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "Nocompress block failed"); + DEBUGLOG(4, "Writing out nocompress block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else if (cSeqsSize == 1) { + cSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "RLE compress block failed"); + DEBUGLOG(4, "Writing out RLE block, size: %zu", cSize); + *dRep = dRepOriginal; /* reset simulated decompression repcode history */ + } else { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + writeBlockHeader(op, cSeqsSize, srcSize, lastBlock); + cSize = ZSTD_blockHeaderSize + cSeqsSize; + DEBUGLOG(4, "Writing out compressed block, size: %zu", cSize); + } + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +/* Struct to keep track of where we are in our recursive calls. */ +typedef struct { + U32* splitLocations; /* Array of split indices */ + size_t idx; /* The current index within splitLocations being worked on */ +} seqStoreSplits; + +#define MIN_SEQUENCES_BLOCK_SPLITTING 300 + +/* Helper function to perform the recursive search for block splits. + * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. + * If advantageous to split, then we recurse down the two sub-blocks. + * If not, or if an error occurred in estimation, then we do not recurse. + * + * Note: The recursion depth is capped by a heuristic minimum number of sequences, + * defined by MIN_SEQUENCES_BLOCK_SPLITTING. + * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). + * In practice, recursion depth usually doesn't go beyond 4. + * + * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. + * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize + * maximum of 128 KB, this value is actually impossible to reach. + */ +static void +ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, + ZSTD_CCtx* zc, const seqStore_t* origSeqStore) +{ + seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; + seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; + seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; + size_t estimatedOriginalSize; + size_t estimatedFirstHalfSize; + size_t estimatedSecondHalfSize; + size_t midIdx = (startIdx + endIdx)/2; + + DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); + assert(endIdx >= startIdx); + if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { + DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); + return; + } + ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); + ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); + ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); + estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc); + estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc); + estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc); + DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", + estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); + if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { + return; + } + if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { + DEBUGLOG(5, "split decided at seqNb:%zu", midIdx); + ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); + splits->splitLocations[splits->idx] = (U32)midIdx; + splits->idx++; + ZSTD_deriveBlockSplitsHelper(splits, midIdx, endIdx, zc, origSeqStore); + } +} + +/* Base recursive function. + * Populates a table with intra-block partition indices that can improve compression ratio. + * + * @return: number of splits made (which equals the size of the partition table - 1). + */ +static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) +{ + seqStoreSplits splits; + splits.splitLocations = partitions; + splits.idx = 0; + if (nbSeq <= 4) { + DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq); + /* Refuse to try and split anything with less than 4 sequences */ + return 0; + } + ZSTD_deriveBlockSplitsHelper(&splits, 0, nbSeq, zc, &zc->seqStore); + splits.splitLocations[splits.idx] = nbSeq; + DEBUGLOG(5, "ZSTD_deriveBlockSplits: final nb partitions: %zu", splits.idx+1); + return splits.idx; +} + +/* ZSTD_compressBlock_splitBlock(): + * Attempts to split a given block into multiple blocks to improve compression ratio. + * + * Returns combined size of all blocks (which includes headers), or a ZSTD error code. + */ +static size_t +ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t blockSize, + U32 lastBlock, U32 nbSeq) +{ + size_t cSize = 0; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + size_t i = 0; + size_t srcBytesTotal = 0; + U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ + seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; + seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; + size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + + /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history + * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two + * separate repcode histories that simulate repcode history on compression and decompression side, + * and use the histories to determine whether we must replace a particular repcode with its raw offset. + * + * 1) cRep gets updated for each partition, regardless of whether the block was emitted as uncompressed + * or RLE. This allows us to retrieve the offset value that an invalid repcode references within + * a nocompress/RLE block. + * 2) dRep gets updated only for compressed partitions, and when a repcode gets replaced, will use + * the replacement offset value rather than the original repcode to update the repcode history. + * dRep also will be the final repcode history sent to the next block. + * + * See ZSTD_seqStore_resolveOffCodes() for more details. + */ + repcodes_t dRep; + repcodes_t cRep; + ZSTD_memcpy(dRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); + ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t)); + + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + if (numSplits == 0) { + size_t cSizeSingleBlock = + ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, + &dRep, &cRep, + op, dstCapacity, + ip, blockSize, + lastBlock, 0 /* isPartition */); + FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); + assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX); + assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize); + return cSizeSingleBlock; + } + + ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); + for (i = 0; i <= numSplits; ++i) { + size_t cSizeChunk; + U32 const lastPartition = (i == numSplits); + U32 lastBlockEntireSrc = 0; + + size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); + srcBytesTotal += srcBytes; + if (lastPartition) { + /* This is the final partition, need to account for possible last literals */ + srcBytes += blockSize - srcBytesTotal; + lastBlockEntireSrc = lastBlock; + } else { + ZSTD_deriveSeqStoreChunk(nextSeqStore, &zc->seqStore, partitions[i], partitions[i+1]); + } + + cSizeChunk = ZSTD_compressSeqStore_singleBlock(zc, currSeqStore, + &dRep, &cRep, + op, dstCapacity, + ip, srcBytes, + lastBlockEntireSrc, 1 /* isPartition */); + DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size", + ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); + FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); + + ip += srcBytes; + op += cSizeChunk; + dstCapacity -= cSizeChunk; + cSize += cSizeChunk; + *currSeqStore = *nextSeqStore; + assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize); + } + /* cRep and dRep may have diverged during the compression. + * If so, we use the dRep repcodes for the next block. + */ + ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t)); + return cSize; +} + +static size_t +ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 lastBlock) +{ + U32 nbSeq; + size_t cSize; + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); + assert(zc->appliedParams.useBlockSplitter == ZSTD_ps_enable); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block"); + return cSize; + } + nbSeq = (U32)(zc->seqStore.sequences - zc->seqStore.sequencesStart); + } + + cSize = ZSTD_compressBlock_splitBlock_internal(zc, dst, dstCapacity, src, srcSize, lastBlock, nbSeq); + FORWARD_IF_ERROR(cSize, "Splitting blocks failed!"); + return cSize; +} + +static size_t +ZSTD_compressBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, U32 frame) +{ + /* This is an estimated upper bound for the length of an rle block. + * This isn't the actual upper bound. + * Finding the real threshold needs further investigation. + */ + const U32 rleMaxLength = 25; + size_t cSize; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, + (unsigned)zc->blockState.matchState.nextToUpdate); + + { const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + if (bss == ZSTDbss_noCompress) { cSize = 0; goto out; } + } + + if (zc->seqCollector.collectSequences) { + ZSTD_copyBlockSequences(zc); + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return 0; + } + + /* encode sequences and literals */ + cSize = ZSTD_entropyCompressSeqStore(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + dst, dstCapacity, + srcSize, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); + + if (frame && + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + cSize < rleMaxLength && + ZSTD_isRLE(ip, srcSize)) + { + cSize = 1; + op[0] = ip[0]; + } + +out: + if (!ZSTD_isError(cSize) && cSize > 1) { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + } + /* We check that dictionaries have offset codes available for the first + * block. After the first block, the offcode table might not have large + * enough codes to represent the offsets in the data. + */ + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const size_t bss, U32 lastBlock) +{ + DEBUGLOG(6, "Attempting ZSTD_compressSuperBlock()"); + if (bss == ZSTDbss_compress) { + if (/* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + !zc->isFirstBlock && + ZSTD_maybeRLE(&zc->seqStore) && + ZSTD_isRLE((BYTE const*)src, srcSize)) + { + return ZSTD_rleCompressBlock(dst, dstCapacity, *(BYTE const*)src, srcSize, lastBlock); + } + /* Attempt superblock compression. + * + * Note that compressed size of ZSTD_compressSuperBlock() is not bound by the + * standard ZSTD_compressBound(). This is a problem, because even if we have + * space now, taking an extra byte now could cause us to run out of space later + * and violate ZSTD_compressBound(). + * + * Define blockBound(blockSize) = blockSize + ZSTD_blockHeaderSize. + * + * In order to respect ZSTD_compressBound() we must attempt to emit a raw + * uncompressed block in these cases: + * * cSize == 0: Return code for an uncompressed block. + * * cSize == dstSize_tooSmall: We may have expanded beyond blockBound(srcSize). + * ZSTD_noCompressBlock() will return dstSize_tooSmall if we are really out of + * output space. + * * cSize >= blockBound(srcSize): We have expanded the block too much so + * emit an uncompressed block. + */ + { size_t const cSize = + ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + if (cSize != ERROR(dstSize_tooSmall)) { + size_t const maxCSize = + srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); + if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { + ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); + return cSize; + } + } + } + } /* if (bss == ZSTDbss_compress)*/ + + DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); + /* Superblock compression failed, attempt to emit a single no compress block. + * The decoder will be able to stream this block since it is uncompressed. + */ + return ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); +} + +static size_t ZSTD_compressBlock_targetCBlockSize(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastBlock) +{ + size_t cSize = 0; + const size_t bss = ZSTD_buildSeqStore(zc, src, srcSize); + DEBUGLOG(5, "ZSTD_compressBlock_targetCBlockSize (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u, srcSize=%zu)", + (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate, srcSize); + FORWARD_IF_ERROR(bss, "ZSTD_buildSeqStore failed"); + + cSize = ZSTD_compressBlock_targetCBlockSize_body(zc, dst, dstCapacity, src, srcSize, bss, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize_body failed"); + + if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + return cSize; +} + +static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + void const* ip, + void const* iend) +{ + U32 const cycleLog = ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy); + U32 const maxDist = (U32)1 << params->cParams.windowLog; + if (ZSTD_window_needOverflowCorrection(ms->window, cycleLog, maxDist, ms->loadedDictEnd, ip, iend)) { + U32 const correction = ZSTD_window_correctOverflow(&ms->window, cycleLog, maxDist, ip); + ZSTD_STATIC_ASSERT(ZSTD_CHAINLOG_MAX <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_32 <= 30); + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + ZSTD_cwksp_mark_tables_dirty(ws); + ZSTD_reduceIndex(ms, params, correction); + ZSTD_cwksp_mark_tables_clean(ws); + if (ms->nextToUpdate < correction) ms->nextToUpdate = 0; + else ms->nextToUpdate -= correction; + /* invalidate dictionaries on overflow correction */ + ms->loadedDictEnd = 0; + ms->dictMatchState = NULL; + } +} + +/*! ZSTD_compress_frameChunk() : +* Compress a chunk of data into one or multiple blocks. +* All blocks will be terminated, all input will be consumed. +* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. +* Frame is supposed already started (header already produced) +* @return : compressed size, or an error code +*/ +static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; + + assert(cctx->appliedParams.cParams.windowLog <= ZSTD_WINDOWLOG_MAX); + + DEBUGLOG(4, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); + if (cctx->appliedParams.fParams.checksumFlag && srcSize) + XXH64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1, + dstSize_tooSmall, + "not enough space to store compressed block"); + if (remaining < blockSize) blockSize = remaining; + + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, ip, ip + blockSize); + ZSTD_checkDictValidity(&ms->window, ip + blockSize, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + ZSTD_window_enforceMaxDist(&ms->window, ip, maxDist, &ms->loadedDictEnd, &ms->dictMatchState); + + /* Ensure hash/chain table insertion resumes no sooner than lowlimit */ + if (ms->nextToUpdate < ms->window.lowLimit) ms->nextToUpdate = ms->window.lowLimit; + + { size_t cSize; + if (ZSTD_useTargetCBlockSize(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_targetCBlockSize(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_targetCBlockSize failed"); + assert(cSize > 0); + assert(cSize <= blockSize + ZSTD_blockHeaderSize); + } else if (ZSTD_blockSplitterEnabled(&cctx->appliedParams)) { + cSize = ZSTD_compressBlock_splitBlock(cctx, op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_splitBlock failed"); + assert(cSize > 0 || cctx->seqCollector.collectSequences == 1); + } else { + cSize = ZSTD_compressBlock_internal(cctx, + op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, + ip, blockSize, 1 /* frame */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressBlock_internal failed"); + + if (cSize == 0) { /* block is not compressible */ + cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + } else { + U32 const cBlockHeader = cSize == 1 ? + lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : + lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader); + cSize += ZSTD_blockHeaderSize; + } + } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ + + + ip += blockSize; + assert(remaining >= blockSize); + remaining -= blockSize; + op += cSize; + assert(dstCapacity >= cSize); + dstCapacity -= cSize; + cctx->isFirstBlock = 0; + DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", + (unsigned)cSize); + } } + + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; + return (size_t)(op-ostart); +} + + +static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, U32 dictID) +{ BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCodeLength = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const dictIDSizeCode = params->fParams.noDictIDFlag ? 0 : dictIDSizeCodeLength; /* 0-3 */ + U32 const checksumFlag = params->fParams.checksumFlag>0; + U32 const windowSize = (U32)1 << params->cParams.windowLog; + U32 const singleSegment = params->fParams.contentSizeFlag && (windowSize >= pledgedSrcSize); + BYTE const windowLogByte = (BYTE)((params->cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params->fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : 0; /* 0-3 */ + BYTE const frameHeaderDescriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + size_t pos=0; + + assert(!(params->fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); + RETURN_ERROR_IF(dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX, dstSize_tooSmall, + "dst buf is too small to fit worst-case frame header size."); + DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", + !params->fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); + if (params->format == ZSTD_f_zstd1) { + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + pos = 4; + } + op[pos++] = frameHeaderDescriptionByte; + if (!singleSegment) op[pos++] = windowLogByte; + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } + switch(fcsCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; + } + return pos; +} + +/* ZSTD_writeSkippableFrame_advanced() : + * Writes out a skippable frame with the specified magic number variant (16 are supported), + * from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15, and the desired source data. + * + * Returns the total number of bytes written, or a ZSTD error code. + */ +size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant) { + BYTE* op = (BYTE*)dst; + RETURN_ERROR_IF(dstCapacity < srcSize + ZSTD_SKIPPABLEHEADERSIZE /* Skippable frame overhead */, + dstSize_tooSmall, "Not enough room for skippable frame"); + RETURN_ERROR_IF(srcSize > (unsigned)0xFFFFFFFF, srcSize_wrong, "Src size too large for skippable frame"); + RETURN_ERROR_IF(magicVariant > 15, parameter_outOfBound, "Skippable frame magic number variant not supported"); + + MEM_writeLE32(op, (U32)(ZSTD_MAGIC_SKIPPABLE_START + magicVariant)); + MEM_writeLE32(op+4, (U32)srcSize); + ZSTD_memcpy(op+8, src, srcSize); + return srcSize + ZSTD_SKIPPABLEHEADERSIZE; +} + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small (stage != ZSTDcs_init, stage_wrong, + "wrong cctx stage"); + RETURN_ERROR_IF(cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable, + parameter_unsupported, + "incompatible with ldm"); + cctx->externSeqStore.seq = seq; + cctx->externSeqStore.size = nbSeq; + cctx->externSeqStore.capacity = nbSeq; + cctx->externSeqStore.pos = 0; + cctx->externSeqStore.posInSequence = 0; + return 0; +} + + +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, U32 lastFrameChunk) +{ + ZSTD_matchState_t* const ms = &cctx->blockState.matchState; + size_t fhSize = 0; + + DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", + cctx->stage, (unsigned)srcSize); + RETURN_ERROR_IF(cctx->stage==ZSTDcs_created, stage_wrong, + "missing init (ZSTD_compressBegin)"); + + if (frame && (cctx->stage==ZSTDcs_init)) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, + cctx->pledgedSrcSizePlusOne-1, cctx->dictID); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + assert(fhSize <= dstCapacity); + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (!srcSize) return fhSize; /* do not generate an empty block if no input */ + + if (!ZSTD_window_update(&ms->window, src, srcSize, ms->forceNonContiguous)) { + ms->forceNonContiguous = 0; + ms->nextToUpdate = ms->window.dictLimit; + } + if (cctx->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_window_update(&cctx->ldmState.window, src, srcSize, /* forceNonContiguous */ 0); + } + + if (!frame) { + /* overflow check and correction for block mode */ + ZSTD_overflowCorrectIfNeeded( + ms, &cctx->workspace, &cctx->appliedParams, + src, (BYTE const*)src + srcSize); + } + + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); + { size_t const cSize = frame ? + ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize, 0 /* frame */); + FORWARD_IF_ERROR(cSize, "%s", frame ? "ZSTD_compress_frameChunk failed" : "ZSTD_compressBlock_internal failed"); + cctx->consumedSrcSize += srcSize; + cctx->producedCSize += (cSize + fhSize); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + RETURN_ERROR_IF( + cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize >= %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + return cSize + fhSize; + } +} + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); +} + +/* NOTE: Must just wrap ZSTD_compressContinue_public() */ +size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize); +} + +static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx) +{ + ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; + assert(!ZSTD_checkCParams(cParams)); + return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); +} + +/* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */ +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + return ZSTD_getBlockSize_deprecated(cctx); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); + { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx); + RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } + + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_loadDictionaryContent() : + * @return : 0, or an error code + */ +static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* src, size_t srcSize, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; + + /* Assert that the ms params match the params we're being given */ + ZSTD_assertEqualCParams(params->cParams, ms->cParams); + + { /* Ensure large dictionaries can't cause index overflow */ + + /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. + * Dictionaries right at the edge will immediately trigger overflow + * correction, but I don't want to insert extra constraints here. + */ + U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + + int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(¶ms->cParams); + if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) { + /* Some dictionary matchfinders in zstd use "short cache", + * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each + * CDict hashtable entry as a tag rather than as part of an index. + * When short cache is used, we need to truncate the dictionary + * so that its indices don't overlap with the tag. */ + U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); + assert(!loadLdmDict); + } + + /* If the dictionary is too large, only load the suffix of the dictionary. */ + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + if (srcSize > ZSTD_CHUNKSIZE_MAX) { + /* We must have cleared our windows when our source is this large. */ + assert(ZSTD_window_isEmpty(ms->window)); + if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window)); + } + ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); + + DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); + + if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */ + ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); + ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); + } + + /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */ + if (params->cParams.strategy < ZSTD_btultra) { + U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28); + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + ms->nextToUpdate = (U32)(ip - ms->window.base); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; + + if (srcSize <= HASH_READ_SIZE) return 0; + + ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); + + switch(params->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, dtlm, tfp); + break; + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + assert(srcSize >= HASH_READ_SIZE); + if (ms->dedicatedDictSearch) { + assert(ms->chainTable != NULL); + ZSTD_dedicatedDictSearch_lazy_loadDictionary(ms, iend-HASH_READ_SIZE); + } else { + assert(params->useRowMatchFinder != ZSTD_ps_auto); + if (params->useRowMatchFinder == ZSTD_ps_enable) { + size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ZSTD_row_update(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using row-based hash table for lazy dict"); + } else { + ZSTD_insertAndFindFirstIndex(ms, iend-HASH_READ_SIZE); + DEBUGLOG(4, "Using chain-based hash table for lazy dict"); + } + } + break; + + case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + assert(srcSize >= HASH_READ_SIZE); + ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); + break; + + default: + assert(0); /* not possible : not a valid strategy id */ + } + + ms->nextToUpdate = (U32)(iend - ms->window.base); + return 0; +} + + +/* Dictionaries that assign zero probability to symbols that show up causes problems + * when FSE encoding. Mark dictionaries with zero probability symbols as FSE_repeat_check + * and only dictionaries with 100% valid symbols can be assumed valid. + */ +static FSE_repeat ZSTD_dictNCountRepeat(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) +{ + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) { + return FSE_repeat_check; + } + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) { + return FSE_repeat_check; + } + } + return FSE_repeat_valid; +} + +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize) +{ + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + const BYTE* dictPtr = (const BYTE*)dict; /* skip magic num and dict ID */ + const BYTE* const dictEnd = dictPtr + dictSize; + dictPtr += 8; + bs->entropy.huf.repeatMode = HUF_repeat_check; + + { unsigned maxSymbolValue = 255; + unsigned hasZeroWeights = 1; + size_t const hufHeaderSize = HUF_readCTable((HUF_CElt*)bs->entropy.huf.CTable, &maxSymbolValue, dictPtr, + dictEnd-dictPtr, &hasZeroWeights); + + /* We only set the loaded table as valid if it contains all non-zero + * weights. Otherwise, we set it to check */ + if (!hasZeroWeights) + bs->entropy.huf.repeatMode = HUF_repeat_valid; + + RETURN_ERROR_IF(HUF_isError(hufHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(maxSymbolValue < 255, dictionary_corrupted, ""); + dictPtr += hufHeaderSize; + } + + { unsigned offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + /* fill all offset symbols to avoid garbage at end of table */ + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.offcodeCTable, + offcodeNCount, MaxOff, offcodeLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.matchlengthCTable, + matchlengthNCount, matchlengthMaxValue, matchlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.matchlength_repeatMode = ZSTD_dictNCountRepeat(matchlengthNCount, matchlengthMaxValue, MaxML); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + RETURN_ERROR_IF(FSE_isError(FSE_buildCTable_wksp( + bs->entropy.fse.litlengthCTable, + litlengthNCount, litlengthMaxValue, litlengthLog, + workspace, HUF_WORKSPACE_SIZE)), + dictionary_corrupted, ""); + bs->entropy.fse.litlength_repeatMode = ZSTD_dictNCountRepeat(litlengthNCount, litlengthMaxValue, MaxLL); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + bs->rep[0] = MEM_readLE32(dictPtr+0); + bs->rep[1] = MEM_readLE32(dictPtr+4); + bs->rep[2] = MEM_readLE32(dictPtr+8); + dictPtr += 12; + + { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + U32 offcodeMax = MaxOff; + if (dictContentSize <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)dictContentSize + 128 KB; /* The maximum offset that must be supported */ + offcodeMax = ZSTD_highbit32(maxOffset); /* Calculate minimum offset code required to represent maxOffset */ + } + /* All offset values <= dictContentSize + 128 KB must be representable for a valid table */ + bs->entropy.fse.offcode_repeatMode = ZSTD_dictNCountRepeat(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)); + + /* All repCodes must be <= dictContentSize and != 0 */ + { U32 u; + for (u=0; u<3; u++) { + RETURN_ERROR_IF(bs->rep[u] == 0, dictionary_corrupted, ""); + RETURN_ERROR_IF(bs->rep[u] > dictContentSize, dictionary_corrupted, ""); + } } } + + return dictPtr - (const BYTE*)dict; +} + +/* Dictionary format : + * See : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#dictionary-format + */ +/*! ZSTD_loadZstdDictionary() : + * @return : dictID, or an error code + * assumptions : magic number supposed already checked + * dictSize supposed >= 8 + */ +static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ZSTD_cwksp* ws, + ZSTD_CCtx_params const* params, + const void* dict, size_t dictSize, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, + void* workspace) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + size_t dictID; + size_t eSize; + ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= 8); + assert(MEM_readLE32(dictPtr) == ZSTD_MAGIC_DICTIONARY); + + dictID = params->fParams.noDictIDFlag ? 0 : MEM_readLE32(dictPtr + 4 /* skip magic number */ ); + eSize = ZSTD_loadCEntropy(bs, workspace, dict, dictSize); + FORWARD_IF_ERROR(eSize, "ZSTD_loadCEntropy failed"); + dictPtr += eSize; + + { + size_t const dictContentSize = (size_t)(dictEnd - dictPtr); + FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); + } + return dictID; +} + +/** ZSTD_compress_insertDictionary() : +* @return : dictID, or an error code */ +static size_t +ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, + ZSTD_matchState_t* ms, + ldmState_t* ls, + ZSTD_cwksp* ws, + const ZSTD_CCtx_params* params, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, + void* workspace) +{ + DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); + if ((dict==NULL) || (dictSize<8)) { + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + return 0; + } + + ZSTD_reset_compressedBlockState(bs); + + /* dict restricted modes */ + if (dictContentType == ZSTD_dct_rawContent) + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp); + + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_auto) { + DEBUGLOG(4, "raw content dictionary detected"); + return ZSTD_loadDictionaryContent( + ms, ls, ws, params, dict, dictSize, dtlm, tfp); + } + RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); + assert(0); /* impossible */ + } + + /* dict as full zstd dictionary */ + return ZSTD_loadZstdDictionary( + bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); +} + +#define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) +#define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) + +/*! ZSTD_compressBegin_internal() : + * Assumption : either @dict OR @cdict (or none) is non-NULL, never both + * @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, U64 pledgedSrcSize, + ZSTD_buffered_policy_e zbuff) +{ + size_t const dictContentSize = cdict ? cdict->dictContentSize : dictSize; +#if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +#endif + DEBUGLOG(4, "ZSTD_compressBegin_internal: wlog=%u", params->cParams.windowLog); + /* params are supposed to be fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if ( (cdict) + && (cdict->dictContentSize > 0) + && ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0) + && (params->attachDictPref != ZSTD_dictForceLoad) ) { + return ZSTD_resetCCtx_usingCDict(cctx, cdict, params, pledgedSrcSize, zbuff); + } + + FORWARD_IF_ERROR( ZSTD_resetCCtx_internal(cctx, params, pledgedSrcSize, + dictContentSize, + ZSTDcrp_makeClean, zbuff) , ""); + { size_t const dictID = cdict ? + ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, + cdict->dictContentSize, cdict->dictContentType, dtlm, + ZSTD_tfp_forCCtx, cctx->entropyWorkspace) + : ZSTD_compress_insertDictionary( + cctx->blockState.prevCBlock, &cctx->blockState.matchState, + &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= UINT_MAX); + cctx->dictID = (U32)dictID; + cctx->dictContentSize = dictContentSize; + } + return 0; +} + +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_compressBegin_advanced_internal: wlog=%u", params->cParams.windowLog); + /* compression parameters verification and optimization */ + FORWARD_IF_ERROR( ZSTD_checkCParams(params->cParams) , ""); + return ZSTD_compressBegin_internal(cctx, + dict, dictSize, dictContentType, dtlm, + cdict, + params, pledgedSrcSize, + ZSTDb_not_buffered); +} + +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compressBegin_advanced_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, + NULL /*cdict*/, + &cctxParams, pledgedSrcSize); +} + +static size_t +ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_CCtx_params cctxParams; + { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); + } + DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); +} + +size_t +ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel); +} + +size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel); +} + + +/*! ZSTD_writeEpilogue() : +* Ends a frame. +* @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + size_t fhSize = 0; + + DEBUGLOG(4, "ZSTD_writeEpilogue"); + RETURN_ERROR_IF(cctx->stage == ZSTDcs_created, stage_wrong, "init missing"); + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, &cctx->appliedParams, 0, 0); + FORWARD_IF_ERROR(fhSize, "ZSTD_writeFrameHeader failed"); + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for epilogue"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op-ostart; +} + +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) +{ +#if ZSTD_TRACE + if (cctx->traceCtx && ZSTD_trace_compress_end != NULL) { + int const streaming = cctx->inBuffSize > 0 || cctx->outBuffSize > 0 || cctx->appliedParams.nbWorkers > 0; + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + trace.dictionaryID = cctx->dictID; + trace.dictionarySize = cctx->dictContentSize; + trace.uncompressedSize = cctx->consumedSrcSize; + trace.compressedSize = cctx->producedCSize + extraCSize; + trace.params = &cctx->appliedParams; + trace.cctx = cctx; + ZSTD_trace_compress_end(cctx->traceCtx, &trace); + } + cctx->traceCtx = 0; +#else + (void)cctx; + (void)extraCSize; +#endif +} + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, + dst, dstCapacity, src, srcSize, + 1 /* frame mode */, 1 /* last chunk */); + FORWARD_IF_ERROR(cSize, "ZSTD_compressContinue_internal failed"); + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + FORWARD_IF_ERROR(endResult, "ZSTD_writeEpilogue failed"); + assert(!(cctx->appliedParams.fParams.contentSizeFlag && cctx->pledgedSrcSizePlusOne == 0)); + if (cctx->pledgedSrcSizePlusOne != 0) { /* control src size */ + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); + DEBUGLOG(4, "end of frame : controlling src size"); + RETURN_ERROR_IF( + cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1, + srcSize_wrong, + "error : pledgedSrcSize = %u, while realSrcSize = %u", + (unsigned)cctx->pledgedSrcSizePlusOne-1, + (unsigned)cctx->consumedSrcSize); + } + ZSTD_CCtx_trace(cctx, endResult); + return cSize + endResult; +} + +/* NOTE: Must just wrap ZSTD_compressEnd_public() */ +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced"); + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, ZSTD_NO_CLEVEL); + return ZSTD_compress_advanced_internal(cctx, + dst, dstCapacity, + src, srcSize, + dict, dictSize, + &cctx->simpleApiParams); +} + +/* Internal */ +size_t ZSTD_compress_advanced_internal( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params) +{ + DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, + params, srcSize, ZSTDb_not_buffered) , ""); + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel) +{ + { + ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, srcSize, dict ? dictSize : 0, ZSTD_cpm_noAttachDict); + assert(params.fParams.contentSizeFlag == 1); + ZSTD_CCtxParams_init_internal(&cctx->simpleApiParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT: compressionLevel); + } + DEBUGLOG(4, "ZSTD_compress_usingDict (srcSize=%u)", (unsigned)srcSize); + return ZSTD_compress_advanced_internal(cctx, dst, dstCapacity, src, srcSize, dict, dictSize, &cctx->simpleApiParams); +} + +size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); + assert(cctx != NULL); + return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel) +{ + size_t result; +#if ZSTD_COMPRESS_HEAPMODE + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + RETURN_ERROR_IF(!cctx, memory_allocation, "ZSTD_createCCtx failed"); + result = ZSTD_compressCCtx(cctx, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtx(cctx); +#else + ZSTD_CCtx ctxBody; + ZSTD_initCCtx(&ctxBody, ZSTD_defaultCMem); + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_freeCCtxContent(&ctxBody); /* can't free ctxBody itself, as it's on stack; free only heap content */ +#endif + return result; +} + + +/* ===== Dictionary API ===== */ + +/*! ZSTD_estimateCDictSize_advanced() : + * Estimate amount of memory that will be needed to create a dictionary with following arguments */ +size_t ZSTD_estimateCDictSize_advanced( + size_t dictSize, ZSTD_compressionParameters cParams, + ZSTD_dictLoadMethod_e dictLoadMethod) +{ + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); + return ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + /* enableDedicatedDictSearch == 1 ensures that CDict estimation will not be too small + * in case we are using DDS with row-hash. */ + + ZSTD_sizeof_matchState(&cParams, ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams), + /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void *)))); +} + +size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + return ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); +} + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); + /* cdict may be in the workspace */ + return (cdict->workspace.workspace == cdict ? 0 : sizeof(*cdict)) + + ZSTD_cwksp_sizeof(&cdict->workspace); +} + +static size_t ZSTD_initCDict_internal( + ZSTD_CDict* cdict, + const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_CCtx_params params) +{ + DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); + assert(!ZSTD_checkCParams(params.cParams)); + cdict->matchState.cParams = params.cParams; + cdict->matchState.dedicatedDictSearch = params.enableDedicatedDictSearch; + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { + cdict->dictContent = dictBuffer; + } else { + void *internalBuffer = ZSTD_cwksp_reserve_object(&cdict->workspace, ZSTD_cwksp_align(dictSize, sizeof(void*))); + RETURN_ERROR_IF(!internalBuffer, memory_allocation, "NULL pointer!"); + cdict->dictContent = internalBuffer; + ZSTD_memcpy(internalBuffer, dictBuffer, dictSize); + } + cdict->dictContentSize = dictSize; + cdict->dictContentType = dictContentType; + + cdict->entropyWorkspace = (U32*)ZSTD_cwksp_reserve_object(&cdict->workspace, HUF_WORKSPACE_SIZE); + + + /* Reset the state to no dictionary */ + ZSTD_reset_compressedBlockState(&cdict->cBlockState); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &cdict->matchState, + &cdict->workspace, + ¶ms.cParams, + params.useRowMatchFinder, + ZSTDcrp_makeClean, + ZSTDirp_reset, + ZSTD_resetTarget_CDict), ""); + /* (Maybe) load the dictionary + * Skips loading the dictionary if it is < 8 bytes. + */ + { params.compressionLevel = ZSTD_CLEVEL_DEFAULT; + params.fParams.contentSizeFlag = 1; + { size_t const dictID = ZSTD_compress_insertDictionary( + &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, + ¶ms, cdict->dictContent, cdict->dictContentSize, + dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); + FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); + assert(dictID <= (size_t)(U32)-1); + cdict->dictID = (U32)dictID; + } + } + + return 0; +} + +static ZSTD_CDict* ZSTD_createCDict_advanced_internal(size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_compressionParameters cParams, + ZSTD_paramSwitch_e useRowMatchFinder, + U32 enableDedicatedDictSearch, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { size_t const workspaceSize = + ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, enableDedicatedDictSearch, /* forCCtx */ 0) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))); + void* const workspace = ZSTD_customMalloc(workspaceSize, customMem); + ZSTD_cwksp ws; + ZSTD_CDict* cdict; + + if (!workspace) { + ZSTD_customFree(workspace, customMem); + return NULL; + } + + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_dynamic_alloc); + + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + assert(cdict != NULL); + ZSTD_cwksp_move(&cdict->workspace, &ws); + cdict->customMem = customMem; + cdict->compressionLevel = ZSTD_NO_CLEVEL; /* signals advanced API usage */ + cdict->useRowMatchFinder = useRowMatchFinder; + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams; + ZSTD_memset(&cctxParams, 0, sizeof(cctxParams)); + ZSTD_CCtxParams_init(&cctxParams, 0); + cctxParams.cParams = cParams; + cctxParams.customMem = customMem; + return ZSTD_createCDict_advanced2( + dictBuffer, dictSize, + dictLoadMethod, dictContentType, + &cctxParams, customMem); +} + +ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* originalCctxParams, + ZSTD_customMem customMem) +{ + ZSTD_CCtx_params cctxParams = *originalCctxParams; + ZSTD_compressionParameters cParams; + ZSTD_CDict* cdict; + + DEBUGLOG(3, "ZSTD_createCDict_advanced2, mode %u", (unsigned)dictContentType); + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + if (cctxParams.enableDedicatedDictSearch) { + cParams = ZSTD_dedicatedDictSearch_getCParams( + cctxParams.compressionLevel, dictSize); + ZSTD_overrideCParams(&cParams, &cctxParams.cParams); + } else { + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + if (!ZSTD_dedicatedDictSearch_isSupported(&cParams)) { + /* Fall back to non-DDSS params */ + cctxParams.enableDedicatedDictSearch = 0; + cParams = ZSTD_getCParamsFromCCtxParams( + &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + } + + DEBUGLOG(3, "ZSTD_createCDict_advanced2: DDS: %u", cctxParams.enableDedicatedDictSearch); + cctxParams.cParams = cParams; + cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + + cdict = ZSTD_createCDict_advanced_internal(dictSize, + dictLoadMethod, cctxParams.cParams, + cctxParams.useRowMatchFinder, cctxParams.enableDedicatedDictSearch, + customMem); + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + cctxParams) )) { + ZSTD_freeCDict(cdict); + return NULL; + } + + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_createCDict); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byRef, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + if (cdict) + cdict->compressionLevel = (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel; + return cdict; +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = cdict->customMem; + int cdictInWorkspace = ZSTD_cwksp_owns_buffer(&cdict->workspace, cdict); + ZSTD_cwksp_free(&cdict->workspace, cMem); + if (!cdictInWorkspace) { + ZSTD_customFree(cdict, cMem); + } + return 0; + } +} + +/*! ZSTD_initStaticCDict_advanced() : + * Generate a digested dictionary in provided memory area. + * workspace: The memory area to emplace the dictionary into. + * Provided pointer must 8-bytes aligned. + * It must outlive dictionary usage. + * workspaceSize: Use ZSTD_estimateCDictSize() + * to determine how large workspace must be. + * cParams : use ZSTD_getCParams() to transform a compression level + * into its relevants cParams. + * @return : pointer to ZSTD_CDict*, or NULL if error (size too small) + * Note : there is no corresponding "free" function. + * Since workspace was allocated externally, it must be freed externally. + */ +const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams) +{ + ZSTD_paramSwitch_e const useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(ZSTD_ps_auto, &cParams); + /* enableDedicatedDictSearch == 1 ensures matchstate is not too small in case this CDict will be used for DDS + row hash */ + size_t const matchStateSize = ZSTD_sizeof_matchState(&cParams, useRowMatchFinder, /* enableDedicatedDictSearch */ 1, /* forCCtx */ 0); + size_t const neededSize = ZSTD_cwksp_alloc_size(sizeof(ZSTD_CDict)) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 + : ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(dictSize, sizeof(void*)))) + + ZSTD_cwksp_alloc_size(HUF_WORKSPACE_SIZE) + + matchStateSize; + ZSTD_CDict* cdict; + ZSTD_CCtx_params params; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + + { + ZSTD_cwksp ws; + ZSTD_cwksp_init(&ws, workspace, workspaceSize, ZSTD_cwksp_static_alloc); + cdict = (ZSTD_CDict*)ZSTD_cwksp_reserve_object(&ws, sizeof(ZSTD_CDict)); + if (cdict == NULL) return NULL; + ZSTD_cwksp_move(&cdict->workspace, &ws); + } + + DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", + (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); + if (workspaceSize < neededSize) return NULL; + + ZSTD_CCtxParams_init(¶ms, 0); + params.cParams = cParams; + params.useRowMatchFinder = useRowMatchFinder; + cdict->useRowMatchFinder = useRowMatchFinder; + cdict->compressionLevel = ZSTD_NO_CLEVEL; + + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + dictLoadMethod, dictContentType, + params) )) + return NULL; + + return cdict; +} + +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict) +{ + assert(cdict != NULL); + return cdict->matchState.cParams; +} + +/*! ZSTD_getDictID_fromCDict() : + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; + return cdict->dictID; +} + +/* ZSTD_compressBegin_usingCDict_internal() : + * Implementation of various ZSTD_compressBegin_usingCDict* functions. + */ +static size_t ZSTD_compressBegin_usingCDict_internal( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + ZSTD_CCtx_params cctxParams; + DEBUGLOG(4, "ZSTD_compressBegin_usingCDict_internal"); + RETURN_ERROR_IF(cdict==NULL, dictionary_wrong, "NULL pointer!"); + /* Initialize the cctxParams from the cdict */ + { + ZSTD_parameters params; + params.fParams = fParams; + params.cParams = ( pledgedSrcSize < ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF + || pledgedSrcSize < cdict->dictContentSize * ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER + || pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN + || cdict->compressionLevel == 0 ) ? + ZSTD_getCParamsFromCDict(cdict) + : ZSTD_getCParams(cdict->compressionLevel, + pledgedSrcSize, + cdict->dictContentSize); + ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, cdict->compressionLevel); + } + /* Increase window log to fit the entire dictionary and source if the + * source size is known. Limit the increase to 19, which is the + * window log for compression level 1 with the largest source size. + */ + if (pledgedSrcSize != ZSTD_CONTENTSIZE_UNKNOWN) { + U32 const limitedSrcSize = (U32)MIN(pledgedSrcSize, 1U << 19); + U32 const limitedSrcLog = limitedSrcSize > 1 ? ZSTD_highbit32(limitedSrcSize - 1) + 1 : 1; + cctxParams.cParams.windowLog = MAX(cctxParams.cParams.windowLog, limitedSrcLog); + } + return ZSTD_compressBegin_internal(cctx, + NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, + cdict, + &cctxParams, pledgedSrcSize, + ZSTDb_not_buffered); +} + + +/* ZSTD_compressBegin_usingCDict_advanced() : + * This function is DEPRECATED. + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_advanced( + ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, + ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize) +{ + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, pledgedSrcSize); +} + +/* ZSTD_compressBegin_usingCDict() : + * cdict must be != NULL */ +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); +} + +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict); +} + +/*! ZSTD_compress_usingCDict_internal(): + * Implementation of various ZSTD_compress_usingCDict* functions. + */ +static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + +/*! ZSTD_compress_usingCDict_advanced(): + * This function is DEPRECATED. + */ +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) +{ + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. + * Note that compression parameters are decided at CDict creation time + * while frame parameters are hardcoded */ +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + ZSTD_frameParameters const fParams = { 1 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; + return ZSTD_compress_usingCDict_internal(cctx, dst, dstCapacity, src, srcSize, cdict, fParams); +} + + + +/* ****************************************************************** +* Streaming +********************************************************************/ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + DEBUGLOG(3, "ZSTD_createCStream"); + return ZSTD_createCStream_advanced(ZSTD_defaultCMem); +} + +ZSTD_CStream* ZSTD_initStaticCStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticCCtx(workspace, workspaceSize); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ /* CStream and CCtx are now same object */ + return ZSTD_createCCtx_advanced(customMem); +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + return ZSTD_freeCCtx(zcs); /* same object */ +} + + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_CStreamOutSize(void) +{ + return ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; +} + +static ZSTD_cParamMode_e ZSTD_getCParamMode(ZSTD_CDict const* cdict, ZSTD_CCtx_params const* params, U64 pledgedSrcSize) +{ + if (cdict != NULL && ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) + return ZSTD_cpm_attachDict; + else + return ZSTD_cpm_noAttachDict; +} + +/* ZSTD_resetCStream(): + * pledgedSrcSize == 0 means "unknown" */ +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +/*! ZSTD_initCStream_internal() : + * Note : for lib/compress only. Used by zstdmt_compress.c. + * Assumption 1 : params are valid + * Assumption 2 : either dict, or cdict, is defined, not both */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_internal"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + assert(!ZSTD_isError(ZSTD_checkCParams(params->cParams))); + zcs->requestedParams = *params; + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + if (dict) { + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + } else { + /* Dictionary is cleared if !cdict */ + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + } + return 0; +} + +/* ZSTD_initCStream_usingCDict_advanced() : + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters */ +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + zcs->requestedParams.fParams = fParams; + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingCDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, cdict) , ""); + return 0; +} + + +/* ZSTD_initCStream_advanced() : + * pledgedSrcSize must be exact. + * if srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * dict is loaded with default parameters ZSTD_dct_auto and ZSTD_dlm_byCopy. */ +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pss) +{ + /* for compatibility with older programs relying on this behavior. + * Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. + * This line will be removed in the future. + */ + U64 const pledgedSrcSize = (pss==0 && params.fParams.contentSizeFlag==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_advanced"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + FORWARD_IF_ERROR( ZSTD_checkCParams(params.cParams) , ""); + ZSTD_CCtxParams_setZstdParams(&zcs->requestedParams, ¶ms); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_loadDictionary(zcs, dict, dictSize) , ""); + return 0; +} + +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pss) +{ + /* temporary : 0 interpreted as "unknown" during transition period. + * Users willing to specify "unknown" **must** use ZSTD_CONTENTSIZE_UNKNOWN. + * 0 will be interpreted as "empty" in the future. + */ + U64 const pledgedSrcSize = (pss==0) ? ZSTD_CONTENTSIZE_UNKNOWN : pss; + DEBUGLOG(4, "ZSTD_initCStream_srcSize"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize) , ""); + return 0; +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + DEBUGLOG(4, "ZSTD_initCStream"); + FORWARD_IF_ERROR( ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_refCDict(zcs, NULL) , ""); + FORWARD_IF_ERROR( ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel) , ""); + return 0; +} + +/*====== Compression ======*/ + +static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + return cctx->blockSize - cctx->stableIn_notConsumed; + } + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered); + { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSize; + return hintInSize; + } +} + +/** ZSTD_compressStream_generic(): + * internal function for all *compressStream*() variants + * @return : hint size for next input to complete ongoing block */ +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective const flushMode) +{ + const char* const istart = (assert(input != NULL), (const char*)input->src); + const char* const iend = (istart != NULL) ? istart + input->size : istart; + const char* ip = (istart != NULL) ? istart + input->pos : istart; + char* const ostart = (assert(output != NULL), (char*)output->dst); + char* const oend = (ostart != NULL) ? ostart + output->size : ostart; + char* op = (ostart != NULL) ? ostart + output->pos : ostart; + U32 someMoreWork = 1; + + /* check expectations */ + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos); + assert(zcs != NULL); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) { + assert(input->pos >= zcs->stableIn_notConsumed); + input->pos -= zcs->stableIn_notConsumed; + ip -= zcs->stableIn_notConsumed; + zcs->stableIn_notConsumed = 0; + } + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + assert(zcs->inBuff != NULL); + assert(zcs->inBuffSize > 0); + } + if (zcs->appliedParams.outBufferMode == ZSTD_bm_buffered) { + assert(zcs->outBuff != NULL); + assert(zcs->outBuffSize > 0); + } + if (input->src == NULL) assert(input->size == 0); + assert(input->pos <= input->size); + if (output->dst == NULL) assert(output->size == 0); + assert(output->pos <= output->size); + assert((U32)flushMode <= (U32)ZSTD_e_end); + + while (someMoreWork) { + switch(zcs->streamStage) + { + case zcss_init: + RETURN_ERROR(init_missing, "call ZSTD_initCStream() first!"); + + case zcss_load: + if ( (flushMode == ZSTD_e_end) + && ( (size_t)(oend-op) >= ZSTD_compressBound(iend-ip) /* Enough output space */ + || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ + && (zcs->inBuffPos == 0) ) { + /* shortcut to compression pass directly into output buffer */ + size_t const cSize = ZSTD_compressEnd_public(zcs, + op, oend-op, ip, iend-ip); + DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); + FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); + ip = iend; + op += cSize; + zcs->frameEnded = 1; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + someMoreWork = 0; break; + } + /* complete loading into inBuffer in buffered mode */ + if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { + size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy( + zcs->inBuff + zcs->inBuffPos, toLoad, + ip, iend-ip); + zcs->inBuffPos += loaded; + if (ip) ip += loaded; + if ( (flushMode == ZSTD_e_continue) + && (zcs->inBuffPos < zcs->inBuffTarget) ) { + /* not enough input to fill full block : stop here */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (zcs->inBuffPos == zcs->inToCompress) ) { + /* empty */ + someMoreWork = 0; break; + } + } else { + assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + if ( (flushMode == ZSTD_e_continue) + && ( (size_t)(iend - ip) < zcs->blockSize) ) { + /* can't compress a full block : stop here */ + zcs->stableIn_notConsumed = (size_t)(iend - ip); + ip = iend; /* pretend to have consumed input */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (ip == iend) ) { + /* empty */ + someMoreWork = 0; break; + } + } + /* compress current block (note : this stage cannot be stopped in the middle) */ + DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); + { int const inputBuffered = (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered); + void* cDst; + size_t cSize; + size_t oSize = oend-op; + size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSize); + if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) + cDst = op; /* compress into output buffer, to skip flush stage */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + if (inputBuffered) { + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); + cSize = lastBlock ? + ZSTD_compressEnd_public(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, + zcs->inBuff + zcs->inToCompress, iSize); + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; + DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", + (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); + if (!lastBlock) + assert(zcs->inBuffTarget <= zcs->inBuffSize); + zcs->inToCompress = zcs->inBuffPos; + } else { /* !inputBuffered, hence ZSTD_bm_stable */ + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend); + cSize = lastBlock ? + ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize); + /* Consume the input prior to error checking to mirror buffered mode. */ + if (ip) ip += iSize; + FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); + zcs->frameEnded = lastBlock; + if (lastBlock) assert(ip == iend); + } + if (cDst == op) { /* no need to flush */ + op += cSize; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed directly in outBuffer"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + } + break; + } + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->streamStage = zcss_flush; /* pass-through to flush stage */ + } + ZSTD_FALLTHROUGH; + case zcss_flush: + DEBUGLOG(5, "flush stage"); + assert(zcs->appliedParams.outBufferMode == ZSTD_bm_buffered); + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, (size_t)(oend-op), + zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", + (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); + if (flushed) + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { + /* flush not fully completed, presumably because dst is too small */ + assert(op==oend); + someMoreWork = 0; + break; + } + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + if (zcs->frameEnded) { + DEBUGLOG(5, "Frame completed on flush"); + someMoreWork = 0; + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + break; + } + zcs->streamStage = zcss_load; + break; + } + + default: /* impossible */ + assert(0); + } + } + + input->pos = ip - istart; + output->pos = op - ostart; + if (zcs->frameEnded) return 0; + return ZSTD_nextInputSizeHint(zcs); +} + +static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers >= 1) { + assert(cctx->mtctx != NULL); + return ZSTDMT_nextInputSizeHint(cctx->mtctx); + } +#endif + return ZSTD_nextInputSizeHint(cctx); + +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + FORWARD_IF_ERROR( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) , ""); + return ZSTD_nextInputSizeHint_MTorST(zcs); +} + +/* After a compression call set the expected input/output buffer. + * This is validated at the start of the next compression call. + */ +static void +ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input) +{ + DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)"); + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + cctx->expectedInBuffer = *input; + } + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + cctx->expectedOutBufferSize = output->size - output->pos; + } +} + +/* Validate that the input/output buffers match the expectations set by + * ZSTD_setBufferExpectations. + */ +static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, + ZSTD_outBuffer const* output, + ZSTD_inBuffer const* input, + ZSTD_EndDirective endOp) +{ + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + ZSTD_inBuffer const expect = cctx->expectedInBuffer; + if (expect.src != input->src || expect.pos != input->pos) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!"); + } + (void)endOp; + if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { + size_t const outBufferSize = output->size - output->pos; + if (cctx->expectedOutBufferSize != outBufferSize) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + } + return 0; +} + +static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, + ZSTD_EndDirective endOp, + size_t inSize) +{ + ZSTD_CCtx_params params = cctx->requestedParams; + ZSTD_prefixDict const prefixDict = cctx->prefixDict; + FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ + ZSTD_memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + if (cctx->cdict && !cctx->localDict.cdict) { + /* Let the cdict's compression level take priority over the requested params. + * But do not take the cdict's compression level if the "cdict" is actually a localDict + * generated from ZSTD_initLocalDict(). + */ + params.compressionLevel = cctx->cdict->compressionLevel; + } + DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */ + + { size_t const dictSize = prefixDict.dict + ? prefixDict.dictSize + : (cctx->cdict ? cctx->cdict->dictContentSize : 0); + ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); + params.cParams = ZSTD_getCParamsFromCCtxParams( + ¶ms, cctx->pledgedSrcSizePlusOne-1, + dictSize, mode); + } + + params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, ¶ms.cParams); + params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); + params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); + params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences); + params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize); + params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel); + +#ifdef ZSTD_MULTITHREAD + /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */ + RETURN_ERROR_IF( + params.useSequenceProducer == 1 && params.nbWorkers >= 1, + parameter_combination_unsupported, + "External sequence producer isn't supported with nbWorkers >= 1" + ); + + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { + params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ + } + if (params.nbWorkers > 0) { +#if ZSTD_TRACE + cctx->traceCtx = (ZSTD_trace_compress_begin != NULL) ? ZSTD_trace_compress_begin(cctx) : 0; +#endif + /* mt context creation */ + if (cctx->mtctx == NULL) { + DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", + params.nbWorkers); + cctx->mtctx = ZSTDMT_createCCtx_advanced((U32)params.nbWorkers, cctx->customMem, cctx->pool); + RETURN_ERROR_IF(cctx->mtctx == NULL, memory_allocation, "NULL pointer!"); + } + /* mt compression */ + DEBUGLOG(4, "call ZSTDMT_initCStream_internal as nbWorkers=%u", params.nbWorkers); + FORWARD_IF_ERROR( ZSTDMT_initCStream_internal( + cctx->mtctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, + cctx->cdict, params, cctx->pledgedSrcSizePlusOne-1) , ""); + cctx->dictID = cctx->cdict ? cctx->cdict->dictID : 0; + cctx->dictContentSize = cctx->cdict ? cctx->cdict->dictContentSize : prefixDict.dictSize; + cctx->consumedSrcSize = 0; + cctx->producedCSize = 0; + cctx->streamStage = zcss_load; + cctx->appliedParams = params; + } else +#endif /* ZSTD_MULTITHREAD */ + { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, + prefixDict.dict, prefixDict.dictSize, prefixDict.dictContentType, ZSTD_dtlm_fast, + cctx->cdict, + ¶ms, pledgedSrcSize, + ZSTDb_buffered) , ""); + assert(cctx->appliedParams.nbWorkers == 0); + cctx->inToCompress = 0; + cctx->inBuffPos = 0; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_buffered) { + /* for small input: avoid automatic flush on reaching end of block, since + * it would require to add a 3-bytes null block to end frame + */ + cctx->inBuffTarget = cctx->blockSize + (cctx->blockSize == pledgedSrcSize); + } else { + cctx->inBuffTarget = 0; + } + cctx->outBuffContentSize = cctx->outBuffFlushedSize = 0; + cctx->streamStage = zcss_load; + cctx->frameEnded = 0; + } + return 0; +} + +/* @return provides a minimum amount of data remaining to be flushed from internal buffers + */ +size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); + /* check conditions */ + RETURN_ERROR_IF(output->pos > output->size, dstSize_tooSmall, "invalid output buffer"); + RETURN_ERROR_IF(input->pos > input->size, srcSize_wrong, "invalid input buffer"); + RETURN_ERROR_IF((U32)endOp > (U32)ZSTD_e_end, parameter_outOfBound, "invalid endDirective"); + assert(cctx != NULL); + + /* transparent initialization stage */ + if (cctx->streamStage == zcss_init) { + size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */ + size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed; + if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */ + && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */ + && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ + if (cctx->stableIn_notConsumed) { /* not the first time */ + /* check stable source guarantees */ + RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer"); + RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos"); + } + /* pretend input was consumed, to give a sense forward progress */ + input->pos = input->size; + /* save stable inBuffer, for later control, and flush/end */ + cctx->expectedInBuffer = *input; + /* but actually input wasn't consumed, so keep track of position from where compression shall resume */ + cctx->stableIn_notConsumed += inputSize; + /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */ + return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */ + } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + } + /* end of transparent initialization stage */ + + FORWARD_IF_ERROR(ZSTD_checkBufferStability(cctx, output, input, endOp), "invalid buffers"); + /* compression stage */ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers > 0) { + size_t flushMin; + if (cctx->cParamsChanged) { + ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); + cctx->cParamsChanged = 0; + } + if (cctx->stableIn_notConsumed) { + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable); + /* some early data was skipped - make it available for consumption */ + assert(input->pos >= cctx->stableIn_notConsumed); + input->pos -= cctx->stableIn_notConsumed; + cctx->stableIn_notConsumed = 0; + } + for (;;) { + size_t const ipos = input->pos; + size_t const opos = output->pos; + flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); + cctx->consumedSrcSize += (U64)(input->pos - ipos); + cctx->producedCSize += (U64)(output->pos - opos); + if ( ZSTD_isError(flushMin) + || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ + if (flushMin == 0) + ZSTD_CCtx_trace(cctx, 0); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + } + FORWARD_IF_ERROR(flushMin, "ZSTDMT_compressStream_generic failed"); + + if (endOp == ZSTD_e_continue) { + /* We only require some progress with ZSTD_e_continue, not maximal progress. + * We're done if we've consumed or produced any bytes, or either buffer is + * full. + */ + if (input->pos != ipos || output->pos != opos || input->pos == input->size || output->pos == output->size) + break; + } else { + assert(endOp == ZSTD_e_flush || endOp == ZSTD_e_end); + /* We require maximal progress. We're done when the flush is complete or the + * output buffer is full. + */ + if (flushMin == 0 || output->pos == output->size) + break; + } + } + DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); + /* Either we don't require maximum forward progress, we've finished the + * flush, or we are out of output space. + */ + assert(endOp == ZSTD_e_continue || flushMin == 0 || output->pos == output->size); + ZSTD_setBufferExpectations(cctx, output, input); + return flushMin; + } +#endif /* ZSTD_MULTITHREAD */ + FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); + DEBUGLOG(5, "completed ZSTD_compressStream2"); + ZSTD_setBufferExpectations(cctx, output, input); + return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ +} + +size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ + { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} + +size_t ZSTD_compress2(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + ZSTD_bufferMode_e const originalInBufferMode = cctx->requestedParams.inBufferMode; + ZSTD_bufferMode_e const originalOutBufferMode = cctx->requestedParams.outBufferMode; + DEBUGLOG(4, "ZSTD_compress2 (srcSize=%u)", (unsigned)srcSize); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + /* Enable stable input/output buffers. */ + cctx->requestedParams.inBufferMode = ZSTD_bm_stable; + cctx->requestedParams.outBufferMode = ZSTD_bm_stable; + { size_t oPos = 0; + size_t iPos = 0; + size_t const result = ZSTD_compressStream2_simpleArgs(cctx, + dst, dstCapacity, &oPos, + src, srcSize, &iPos, + ZSTD_e_end); + /* Reset to the original values. */ + cctx->requestedParams.inBufferMode = originalInBufferMode; + cctx->requestedParams.outBufferMode = originalOutBufferMode; + + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); + if (result != 0) { /* compression not completed, due to lack of output space */ + assert(oPos == dstCapacity); + RETURN_ERROR(dstSize_tooSmall, ""); + } + assert(iPos == srcSize); /* all input is expected consumed */ + return oPos; + } +} + +/* ZSTD_validateSequence() : + * @offCode : is presumed to follow format required by ZSTD_storeSeq() + * @returns a ZSTD error code if sequence is not valid + */ +static size_t +ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch, + size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer) +{ + U32 const windowSize = 1u << windowLog; + /* posInSrc represents the amount of data the decoder would decode up to this point. + * As long as the amount of data decoded is less than or equal to window size, offsets may be + * larger than the total length of output decoded in order to reference the dict, even larger than + * window size. After output surpasses windowSize, we're limited to windowSize offsets again. + */ + size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; + size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4; + RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!"); + /* Validate maxNbSeq is large enough for the given matchLength and minMatch */ + RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch"); + return 0; +} + +/* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ +static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) +{ + U32 offBase = OFFSET_TO_OFFBASE(rawOffset); + + if (!ll0 && rawOffset == rep[0]) { + offBase = REPCODE1_TO_OFFBASE; + } else if (rawOffset == rep[1]) { + offBase = REPCODE_TO_OFFBASE(2 - ll0); + } else if (rawOffset == rep[2]) { + offBase = REPCODE_TO_OFFBASE(3 - ll0); + } else if (ll0 && rawOffset == rep[0] - 1) { + offBase = REPCODE3_TO_OFFBASE; + } + return offBase; +} + +size_t +ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, + ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, + ZSTD_paramSwitch_e externalRepSearch) +{ + U32 idx = seqPos->idx; + U32 const startIdx = idx; + BYTE const* ip = (BYTE const*)(src); + const BYTE* const iend = ip + blockSize; + repcodes_t updatedRepcodes; + U32 dictSize; + + DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize); + + if (cctx->cdict) { + dictSize = (U32)cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = (U32)cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) { + U32 const litLength = inSeqs[idx].litLength; + U32 const matchLength = inSeqs[idx].matchLength; + U32 offBase; + + if (externalRepSearch == ZSTD_ps_disable) { + offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset); + } else { + U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer), + "Sequence validation failed"); + } + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); + ip += matchLength + litLength; + } + + /* If we skipped repcode search while parsing, we need to update repcodes now */ + assert(externalRepSearch != ZSTD_ps_auto); + assert(idx >= startIdx); + if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) { + U32* const rep = updatedRepcodes.rep; + U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */ + + if (lastSeqIdx >= startIdx + 2) { + rep[2] = inSeqs[lastSeqIdx - 2].offset; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else if (lastSeqIdx == startIdx + 1) { + rep[2] = rep[0]; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else { + assert(lastSeqIdx == startIdx); + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = inSeqs[lastSeqIdx].offset; + } + } + + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + if (inSeqs[idx].litLength) { + DEBUGLOG(6, "Storing last literals of size: %u", inSeqs[idx].litLength); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, inSeqs[idx].litLength); + ip += inSeqs[idx].litLength; + seqPos->posInSrc += inSeqs[idx].litLength; + } + RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!"); + seqPos->idx = idx+1; + return 0; +} + +size_t +ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch) +{ + U32 idx = seqPos->idx; + U32 startPosInSequence = seqPos->posInSequence; + U32 endPosInSequence = seqPos->posInSequence + (U32)blockSize; + size_t dictSize; + BYTE const* ip = (BYTE const*)(src); + BYTE const* iend = ip + blockSize; /* May be adjusted if we decide to process fewer than blockSize bytes */ + repcodes_t updatedRepcodes; + U32 bytesAdjustment = 0; + U32 finalMatchSplit = 0; + + /* TODO(embg) support fast parsing mode in noBlockDelim mode */ + (void)externalRepSearch; + + if (cctx->cdict) { + dictSize = cctx->cdict->dictContentSize; + } else if (cctx->prefixDict.dict) { + dictSize = cctx->prefixDict.dictSize; + } else { + dictSize = 0; + } + DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); + while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { + const ZSTD_Sequence currSeq = inSeqs[idx]; + U32 litLength = currSeq.litLength; + U32 matchLength = currSeq.matchLength; + U32 const rawOffset = currSeq.offset; + U32 offBase; + + /* Modify the sequence depending on where endPosInSequence lies */ + if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { + if (startPosInSequence >= litLength) { + startPosInSequence -= litLength; + litLength = 0; + matchLength -= startPosInSequence; + } else { + litLength -= startPosInSequence; + } + /* Move to the next sequence */ + endPosInSequence -= currSeq.litLength + currSeq.matchLength; + startPosInSequence = 0; + } else { + /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence + does not reach the end of the match. So, we have to split the sequence */ + DEBUGLOG(6, "Require a split: diff: %u, idx: %u PIS: %u", + currSeq.litLength + currSeq.matchLength - endPosInSequence, idx, endPosInSequence); + if (endPosInSequence > litLength) { + U32 firstHalfMatchLength; + litLength = startPosInSequence >= litLength ? 0 : litLength - startPosInSequence; + firstHalfMatchLength = endPosInSequence - startPosInSequence - litLength; + if (matchLength > blockSize && firstHalfMatchLength >= cctx->appliedParams.cParams.minMatch) { + /* Only ever split the match if it is larger than the block size */ + U32 secondHalfMatchLength = currSeq.matchLength + currSeq.litLength - endPosInSequence; + if (secondHalfMatchLength < cctx->appliedParams.cParams.minMatch) { + /* Move the endPosInSequence backward so that it creates match of minMatch length */ + endPosInSequence -= cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + bytesAdjustment = cctx->appliedParams.cParams.minMatch - secondHalfMatchLength; + firstHalfMatchLength -= bytesAdjustment; + } + matchLength = firstHalfMatchLength; + /* Flag that we split the last match - after storing the sequence, exit the loop, + but keep the value of endPosInSequence */ + finalMatchSplit = 1; + } else { + /* Move the position in sequence backwards so that we don't split match, and break to store + * the last literals. We use the original currSeq.litLength as a marker for where endPosInSequence + * should go. We prefer to do this whenever it is not necessary to split the match, or if doing so + * would cause the first half of the match to be too small + */ + bytesAdjustment = endPosInSequence - currSeq.litLength; + endPosInSequence = currSeq.litLength; + break; + } + } else { + /* This sequence ends inside the literals, break to store the last literals */ + break; + } + } + /* Check if this offset can be represented with a repcode */ + { U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + + if (cctx->appliedParams.validateSequences) { + seqPos->posInSrc += litLength + matchLength; + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer), + "Sequence validation failed"); + } + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, + "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); + ip += matchLength + litLength; + if (!finalMatchSplit) + idx++; /* Next Sequence */ + } + DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); + assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); + seqPos->idx = idx; + seqPos->posInSequence = endPosInSequence; + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); + + iend -= bytesAdjustment; + if (ip != iend) { + /* Store any last literals */ + U32 lastLLSize = (U32)(iend - ip); + assert(ip <= iend); + DEBUGLOG(6, "Storing last literals of size: %u", lastLLSize); + ZSTD_storeLastLiterals(&cctx->seqStore, ip, lastLLSize); + seqPos->posInSrc += lastLLSize; + } + + return bytesAdjustment; +} + +typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); +static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) +{ + ZSTD_sequenceCopier sequenceCopier = NULL; + assert(ZSTD_cParam_withinBounds(ZSTD_c_blockDelimiters, mode)); + if (mode == ZSTD_sf_explicitBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreExplicitBlockDelim; + } else if (mode == ZSTD_sf_noBlockDelimiters) { + return ZSTD_copySequencesToSeqStoreNoBlockDelim; + } + assert(sequenceCopier != NULL); + return sequenceCopier; +} + +/* Discover the size of next block by searching for the delimiter. + * Note that a block delimiter **must** exist in this mode, + * otherwise it's an input error. + * The block size retrieved will be later compared to ensure it remains within bounds */ +static size_t +blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) +{ + int end = 0; + size_t blockSize = 0; + size_t spos = seqPos.idx; + DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize); + assert(spos <= inSeqsSize); + while (spos < inSeqsSize) { + end = (inSeqs[spos].offset == 0); + blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength; + if (end) { + if (inSeqs[spos].matchLength != 0) + RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0"); + break; + } + spos++; + } + if (!end) + RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter"); + return blockSize; +} + +/* More a "target" block size */ +static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining) +{ + int const lastBlock = (remaining <= blockSize); + return lastBlock ? remaining : blockSize; +} + +static size_t determine_blockSize(ZSTD_sequenceFormat_e mode, + size_t blockSize, size_t remaining, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) +{ + DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); + if (mode == ZSTD_sf_noBlockDelimiters) + return blockSize_noDelimiter(blockSize, remaining); + { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos); + FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters"); + if (explicitBlockSize > blockSize) + RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block"); + if (explicitBlockSize > remaining) + RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source"); + return explicitBlockSize; + } +} + +/* Compress, block-by-block, all of the sequences given. + * + * Returns the cumulative size of all compressed blocks (including their headers), + * otherwise a ZSTD error. + */ +static size_t +ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + size_t cSize = 0; + size_t remaining = srcSize; + ZSTD_sequencePosition seqPos = {0, 0, 0}; + + BYTE const* ip = (BYTE const*)src; + BYTE* op = (BYTE*)dst; + ZSTD_sequenceCopier const sequenceCopier = ZSTD_selectSequenceCopier(cctx->appliedParams.blockDelimiters); + + DEBUGLOG(4, "ZSTD_compressSequences_internal srcSize: %zu, inSeqsSize: %zu", srcSize, inSeqsSize); + /* Special case: empty frame */ + if (remaining == 0) { + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "No room for empty frame block header"); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + cSize += ZSTD_blockHeaderSize; + } + + while (remaining) { + size_t compressedSeqsSize; + size_t cBlockSize; + size_t additionalByteAdjustment; + size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, + cctx->blockSize, remaining, + inSeqs, inSeqsSize, seqPos); + U32 const lastBlock = (blockSize == remaining); + FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size"); + assert(blockSize <= remaining); + ZSTD_resetSeqStore(&cctx->seqStore); + DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize); + + additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes); + FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); + blockSize -= additionalByteAdjustment; + + /* If blocks are too small, emit as a nocompress block */ + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); + DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); + cSize += cBlockSize; + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + continue; + } + + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); + compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, + &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, + &cctx->appliedParams, + op + ZSTD_blockHeaderSize /* Leave space for block header */, dstCapacity - ZSTD_blockHeaderSize, + blockSize, + cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + cctx->bmi2); + FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); + DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); + + if (!cctx->isFirstBlock && + ZSTD_maybeRLE(&cctx->seqStore) && + ZSTD_isRLE(ip, blockSize)) { + /* We don't want to emit our first block as a RLE even if it qualifies because + * doing so will cause the decoder (cli only) to throw a "should consume all input error." + * This is only an issue for zstd <= v1.4.3 + */ + compressedSeqsSize = 1; + } + + if (compressedSeqsSize == 0) { + /* ZSTD_noCompressBlock writes the block header as well */ + cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); + } else if (compressedSeqsSize == 1) { + cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed"); + DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); + } else { + U32 cBlockHeader; + /* Error checking and repcodes update */ + ZSTD_blockState_confirmRepcodesAndEntropyTables(&cctx->blockState); + if (cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) + cctx->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; + + /* Write block header into beginning of block*/ + cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); + MEM_writeLE24(op, cBlockHeader); + cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; + DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); + } + + cSize += cBlockSize; + + if (lastBlock) { + break; + } else { + ip += blockSize; + op += cBlockSize; + remaining -= blockSize; + dstCapacity -= cBlockSize; + cctx->isFirstBlock = 0; + } + DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); + } + + DEBUGLOG(4, "cSize final total: %zu", cSize); + return cSize; +} + +size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize) +{ + BYTE* op = (BYTE*)dst; + size_t cSize = 0; + size_t compressedBlocksSize = 0; + size_t frameHeaderSize = 0; + + /* Transparent initialization stage, same as compressStream2() */ + DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity); + assert(cctx != NULL); + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); + /* Begin writing output, starting with frame header */ + frameHeaderSize = ZSTD_writeFrameHeader(op, dstCapacity, &cctx->appliedParams, srcSize, cctx->dictID); + op += frameHeaderSize; + dstCapacity -= frameHeaderSize; + cSize += frameHeaderSize; + if (cctx->appliedParams.fParams.checksumFlag && srcSize) { + XXH64_update(&cctx->xxhState, src, srcSize); + } + /* cSize includes block header size and compressed sequences size */ + compressedBlocksSize = ZSTD_compressSequences_internal(cctx, + op, dstCapacity, + inSeqs, inSeqsSize, + src, srcSize); + FORWARD_IF_ERROR(compressedBlocksSize, "Compressing blocks failed!"); + cSize += compressedBlocksSize; + dstCapacity -= compressedBlocksSize; + + if (cctx->appliedParams.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + RETURN_ERROR_IF(dstCapacity<4, dstSize_tooSmall, "no room for checksum"); + DEBUGLOG(4, "Write checksum : %08X", (unsigned)checksum); + MEM_writeLE32((char*)dst + cSize, checksum); + cSize += 4; + } + + DEBUGLOG(4, "Final compressed size: %zu", cSize); + return cSize; +} + +/*====== Finalize ======*/ + +static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs) +{ + const ZSTD_inBuffer nullInput = { NULL, 0, 0 }; + const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + return stableInput ? zcs->expectedInBuffer : nullInput; +} + +/*! ZSTD_flushStream() : + * @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + input.size = input.pos; /* do not ingest more input during flush */ + return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); + FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed"); + if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ + /* single thread mode : attempt to calculate remaining to flush more precisely */ + { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; + size_t const checksumSize = (size_t)(zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4); + size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; + DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); + return toFlush; + } +} + + +/*-===== Pre-defined compression levels =====-*/ +#include "clevels.h" + +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } +int ZSTD_minCLevel(void) { return (int)-ZSTD_TARGETLENGTH_MAX; } +int ZSTD_defaultCLevel(void) { return ZSTD_CLEVEL_DEFAULT; } + +static ZSTD_compressionParameters ZSTD_dedicatedDictSearch_getCParams(int const compressionLevel, size_t const dictSize) +{ + ZSTD_compressionParameters cParams = ZSTD_getCParams_internal(compressionLevel, 0, dictSize, ZSTD_cpm_createCDict); + switch (cParams.strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams.hashLog += ZSTD_LAZY_DDSS_BUCKET_LOG; + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } + return cParams; +} + +static int ZSTD_dedicatedDictSearch_isSupported( + ZSTD_compressionParameters const* cParams) +{ + return (cParams->strategy >= ZSTD_greedy) + && (cParams->strategy <= ZSTD_lazy2) + && (cParams->hashLog > cParams->chainLog) + && (cParams->chainLog <= 24); +} + +/** + * Reverses the adjustment applied to cparams when enabling dedicated dict + * search. This is used to recover the params set to be used in the working + * context. (Otherwise, those tables would also grow.) + */ +static void ZSTD_dedicatedDictSearch_revertCParams( + ZSTD_compressionParameters* cParams) { + switch (cParams->strategy) { + case ZSTD_fast: + case ZSTD_dfast: + break; + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + cParams->hashLog -= ZSTD_LAZY_DDSS_BUCKET_LOG; + if (cParams->hashLog < ZSTD_HASHLOG_MIN) { + cParams->hashLog = ZSTD_HASHLOG_MIN; + } + break; + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + } +} + +static U64 ZSTD_getCParamRowSize(U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + switch (mode) { + case ZSTD_cpm_unknown: + case ZSTD_cpm_noAttachDict: + case ZSTD_cpm_createCDict: + break; + case ZSTD_cpm_attachDict: + dictSize = 0; + break; + default: + assert(0); + break; + } + { int const unknown = srcSizeHint == ZSTD_CONTENTSIZE_UNKNOWN; + size_t const addedSize = unknown && dictSize > 0 ? 500 : 0; + return unknown && dictSize == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : srcSizeHint+dictSize+addedSize; + } +} + +/*! ZSTD_getCParams_internal() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Note: srcSizeHint 0 means 0, use ZSTD_CONTENTSIZE_UNKNOWN for unknown. + * Use dictSize == 0 for unknown or unused. + * Note: `mode` controls how we treat the `dictSize`. See docs for `ZSTD_cParamMode_e`. */ +static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) +{ + U64 const rSize = ZSTD_getCParamRowSize(srcSizeHint, dictSize, mode); + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); + int row; + DEBUGLOG(5, "ZSTD_getCParams_internal (cLevel=%i)", compressionLevel); + + /* row */ + if (compressionLevel == 0) row = ZSTD_CLEVEL_DEFAULT; /* 0 == default */ + else if (compressionLevel < 0) row = 0; /* entry 0 is baseline for fast mode */ + else if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; + else row = compressionLevel; + + { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; + DEBUGLOG(5, "ZSTD_getCParams_internal selected tableID: %u row: %u strat: %u", tableID, row, (U32)cp.strategy); + /* acceleration factor */ + if (compressionLevel < 0) { + int const clampedCompressionLevel = MAX(ZSTD_minCLevel(), compressionLevel); + cp.targetLength = (unsigned)(-clampedCompressionLevel); + } + /* refine parameters based on srcSize & dictSize */ + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); + } +} + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level, srcSize and dictSize. + * Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) +{ + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +static ZSTD_parameters ZSTD_getParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode) { + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams_internal(compressionLevel, srcSizeHint, dictSize, mode); + DEBUGLOG(5, "ZSTD_getParams (cLevel=%i)", compressionLevel); + ZSTD_memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + params.fParams.contentSizeFlag = 1; + return params; +} + +/*! ZSTD_getParams() : + * same idea as ZSTD_getCParams() + * @return a `ZSTD_parameters` structure (instead of `ZSTD_compressionParameters`). + * Fields of `ZSTD_frameParameters` are set to default values */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize) { + if (srcSizeHint == 0) srcSizeHint = ZSTD_CONTENTSIZE_UNKNOWN; + return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); +} + +void ZSTD_registerSequenceProducer( + ZSTD_CCtx* zc, void* mState, + ZSTD_sequenceProducer_F* mFinder +) { + if (mFinder != NULL) { + ZSTD_externalMatchCtx emctx; + emctx.mState = mState; + emctx.mFinder = mFinder; + emctx.seqBuffer = NULL; + emctx.seqBufferCapacity = 0; + zc->externalMatchCtx = emctx; + zc->requestedParams.useSequenceProducer = 1; + } else { + ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx)); + zc->requestedParams.useSequenceProducer = 0; + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_internal.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_internal.h new file mode 100644 index 000000000..10f68d010 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_internal.h @@ -0,0 +1,1532 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This header contains definitions + * that shall **only** be used by modules within lib/compress. + */ + +#ifndef ZSTD_COMPRESS_H +#define ZSTD_COMPRESS_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/zstd_internal.h" +#include "zstd_cwksp.h" +#ifdef ZSTD_MULTITHREAD +# include "zstdmt_compress.h" +#endif +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ +#define kSearchStrength 8 +#define HASH_READ_SIZE 8 +#define ZSTD_DUBT_UNSORTED_MARK 1 /* For btlazy2 strategy, index ZSTD_DUBT_UNSORTED_MARK==1 means "unsorted". + It could be confused for a real successor at index "1", if sorted as larger than its predecessor. + It's not a big deal though : candidate will just be sorted again. + Additionally, candidate position 1 will be lost. + But candidate 1 cannot hide a large tree of candidates, so it's a minimal loss. + The benefit is that ZSTD_DUBT_UNSORTED_MARK cannot be mishandled after table re-use with a different strategy. + This constant is required by ZSTD_compressBlock_btlazy2() and ZSTD_reduceTable_internal() */ + + +/*-************************************* +* Context memory management +***************************************/ +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; +typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; + +typedef struct ZSTD_prefixDict_s { + const void* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; +} ZSTD_prefixDict; + +typedef struct { + void* dictBuffer; + void const* dict; + size_t dictSize; + ZSTD_dictContentType_e dictContentType; + ZSTD_CDict* cdict; +} ZSTD_localDict; + +typedef struct { + HUF_CElt CTable[HUF_CTABLE_SIZE_ST(255)]; + HUF_repeat repeatMode; +} ZSTD_hufCTables_t; + +typedef struct { + FSE_CTable offcodeCTable[FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable[FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + FSE_repeat offcode_repeatMode; + FSE_repeat matchlength_repeatMode; + FSE_repeat litlength_repeatMode; +} ZSTD_fseCTables_t; + +typedef struct { + ZSTD_hufCTables_t huf; + ZSTD_fseCTables_t fse; +} ZSTD_entropyCTables_t; + +/*********************************************** +* Entropy buffer statistics structs and funcs * +***********************************************/ +/** ZSTD_hufCTablesMetadata_t : + * Stores Literals Block Type for a super-block in hType, and + * huffman tree description in hufDesBuffer. + * hufDesSize refers to the size of huffman tree description in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_literals() */ +typedef struct { + symbolEncodingType_e hType; + BYTE hufDesBuffer[ZSTD_MAX_HUF_HEADER_SIZE]; + size_t hufDesSize; +} ZSTD_hufCTablesMetadata_t; + +/** ZSTD_fseCTablesMetadata_t : + * Stores symbol compression modes for a super-block in {ll, ol, ml}Type, and + * fse tables in fseTablesBuffer. + * fseTablesSize refers to the size of fse tables in bytes. + * This metadata is populated in ZSTD_buildBlockEntropyStats_sequences() */ +typedef struct { + symbolEncodingType_e llType; + symbolEncodingType_e ofType; + symbolEncodingType_e mlType; + BYTE fseTablesBuffer[ZSTD_MAX_FSE_HEADERS_SIZE]; + size_t fseTablesSize; + size_t lastCountSize; /* This is to account for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ +} ZSTD_fseCTablesMetadata_t; + +typedef struct { + ZSTD_hufCTablesMetadata_t hufMetadata; + ZSTD_fseCTablesMetadata_t fseMetadata; +} ZSTD_entropyCTablesMetadata_t; + +/** ZSTD_buildBlockEntropyStats() : + * Builds entropy for the block. + * @return : 0 on success or error code */ +size_t ZSTD_buildBlockEntropyStats( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); + +/********************************* +* Compression internals structs * +*********************************/ + +typedef struct { + U32 off; /* Offset sumtype code for the match, using ZSTD_storeSeq() format */ + U32 len; /* Raw length of match */ +} ZSTD_match_t; + +typedef struct { + U32 offset; /* Offset of sequence */ + U32 litLength; /* Length of literals prior to match */ + U32 matchLength; /* Raw length of match */ +} rawSeq; + +typedef struct { + rawSeq* seq; /* The start of the sequences */ + size_t pos; /* The index in seq where reading stopped. pos <= size. */ + size_t posInSequence; /* The position within the sequence at seq[pos] where reading + stopped. posInSequence <= seq[pos].litLength + seq[pos].matchLength */ + size_t size; /* The number of sequences. <= capacity. */ + size_t capacity; /* The capacity starting from `seq` pointer */ +} rawSeqStore_t; + +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_sequencePosition; + +UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; + +typedef struct { + int price; + U32 off; + U32 mlen; + U32 litlen; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_optimal_t; + +typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; + +typedef struct { + /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ + unsigned* litFreq; /* table of literals statistics, of size 256 */ + unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ + unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ + unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ + ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */ + ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */ + + U32 litSum; /* nb of literals */ + U32 litLengthSum; /* nb of litLength codes */ + U32 matchLengthSum; /* nb of matchLength codes */ + U32 offCodeSum; /* nb of offset codes */ + U32 litSumBasePrice; /* to compare to log2(litfreq) */ + U32 litLengthSumBasePrice; /* to compare to log2(llfreq) */ + U32 matchLengthSumBasePrice;/* to compare to log2(mlfreq) */ + U32 offCodeSumBasePrice; /* to compare to log2(offreq) */ + ZSTD_OptPrice_e priceType; /* prices can be determined dynamically, or follow a pre-defined cost structure */ + const ZSTD_entropyCTables_t* symbolCosts; /* pre-calculated dictionary statistics */ + ZSTD_paramSwitch_e literalCompressionMode; +} optState_t; + +typedef struct { + ZSTD_entropyCTables_t entropy; + U32 rep[ZSTD_REP_NUM]; +} ZSTD_compressedBlockState_t; + +typedef struct { + BYTE const* nextSrc; /* next block here to continue on current prefix */ + BYTE const* base; /* All regular indexes relative to this position */ + BYTE const* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more valid data */ + U32 nbOverflowCorrections; /* Number of times overflow correction has run since + * ZSTD_window_init(). Useful for debugging coredumps + * and for ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY. + */ +} ZSTD_window_t; + +#define ZSTD_WINDOW_START_INDEX 2 + +typedef struct ZSTD_matchState_t ZSTD_matchState_t; + +#define ZSTD_ROW_HASH_CACHE_SIZE 8 /* Size of prefetching hash cache for row-based matchfinder */ + +struct ZSTD_matchState_t { + ZSTD_window_t window; /* State for window round buffer management */ + U32 loadedDictEnd; /* index of end of dictionary, within context's referential. + * When loadedDictEnd != 0, a dictionary is in use, and still valid. + * This relies on a mechanism to set loadedDictEnd=0 when dictionary is no longer within distance. + * Such mechanism is provided within ZSTD_window_enforceMaxDist() and ZSTD_checkDictValidity(). + * When dict referential is copied into active context (i.e. not attached), + * loadedDictEnd == dictSize, since referential starts from zero. + */ + U32 nextToUpdate; /* index from which to continue table update */ + U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ + + U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ + BYTE* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ + U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ + U64 hashSalt; /* For row-based matchFinder: salts the hash for re-use of tag table */ + U32 hashSaltEntropy; /* For row-based matchFinder: collects entropy for salt generation */ + + U32* hashTable; + U32* hashTable3; + U32* chainTable; + + U32 forceNonContiguous; /* Non-zero if we should force non-contiguous load for the next window update. */ + + int dedicatedDictSearch; /* Indicates whether this matchState is using the + * dedicated dictionary search structure. + */ + optState_t opt; /* optimal parser state */ + const ZSTD_matchState_t* dictMatchState; + ZSTD_compressionParameters cParams; + const rawSeqStore_t* ldmSeqStore; + + /* Controls prefetching in some dictMatchState matchfinders. + * This behavior is controlled from the cctx ms. + * This parameter has no effect in the cdict ms. */ + int prefetchCDictTables; + + /* When == 0, lazy match finders insert every position. + * When != 0, lazy match finders only insert positions they search. + * This allows them to skip much faster over incompressible data, + * at a small cost to compression ratio. + */ + int lazySkipping; +}; + +typedef struct { + ZSTD_compressedBlockState_t* prevCBlock; + ZSTD_compressedBlockState_t* nextCBlock; + ZSTD_matchState_t matchState; +} ZSTD_blockState_t; + +typedef struct { + U32 offset; + U32 checksum; +} ldmEntry_t; + +typedef struct { + BYTE const* split; + U32 hash; + U32 checksum; + ldmEntry_t* bucket; +} ldmMatchCandidate_t; + +#define LDM_BATCH_SIZE 64 + +typedef struct { + ZSTD_window_t window; /* State for the window round buffer management */ + ldmEntry_t* hashTable; + U32 loadedDictEnd; + BYTE* bucketOffsets; /* Next position in bucket to insert entry */ + size_t splitIndices[LDM_BATCH_SIZE]; + ldmMatchCandidate_t matchCandidates[LDM_BATCH_SIZE]; +} ldmState_t; + +typedef struct { + ZSTD_paramSwitch_e enableLdm; /* ZSTD_ps_enable to enable LDM. ZSTD_ps_auto by default */ + U32 hashLog; /* Log size of hashTable */ + U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ + U32 minMatchLength; /* Minimum match length */ + U32 hashRateLog; /* Log number of entries to skip */ + U32 windowLog; /* Window log for the LDM */ +} ldmParams_t; + +typedef struct { + int collectSequences; + ZSTD_Sequence* seqStart; + size_t seqIndex; + size_t maxSequences; +} SeqCollector; + +struct ZSTD_CCtx_params_s { + ZSTD_format_e format; + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; + + int compressionLevel; + int forceWindow; /* force back-references to respect limit of + * 1< 63) ? ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; +} + +/* ZSTD_MLcode() : + * note : mlBase = matchLength - MINMATCH; + * because it's the format it's stored in seqStore->sequences */ +MEM_STATIC U32 ZSTD_MLcode(U32 mlBase) +{ + static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + static const U32 ML_deltaCode = 36; + return (mlBase > 127) ? ZSTD_highbit32(mlBase) + ML_deltaCode : ML_Code[mlBase]; +} + +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +/* ZSTD_noCompressBlock() : + * Writes uncompressed block to dst buffer from given src. + * Returns the size of the block */ +MEM_STATIC size_t +ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +{ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity); + RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, + dstSize_tooSmall, "dst buf too small for uncompressed block"); + MEM_writeLE24(dst, cBlockHeader24); + ZSTD_memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + return ZSTD_blockHeaderSize + srcSize; +} + +MEM_STATIC size_t +ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +{ + BYTE* const op = (BYTE*)dst; + U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); + RETURN_ERROR_IF(dstCapacity < 4, dstSize_tooSmall, ""); + MEM_writeLE24(op, cBlockHeader); + op[3] = src; + return 4; +} + + +/* ZSTD_minGain() : + * minimum compression required + * to generate a compress block or a compressed literals section. + * note : use same formula for both situations */ +MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) +{ + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); + return (srcSize >> minlog) + 2; +} + +MEM_STATIC int ZSTD_literalsCompressionIsDisabled(const ZSTD_CCtx_params* cctxParams) +{ + switch (cctxParams->literalCompressionMode) { + case ZSTD_ps_enable: + return 0; + case ZSTD_ps_disable: + return 1; + default: + assert(0 /* impossible: pre-validated */); + ZSTD_FALLTHROUGH; + case ZSTD_ps_auto: + return (cctxParams->cParams.strategy == ZSTD_fast) && (cctxParams->cParams.targetLength > 0); + } +} + +/*! ZSTD_safecopyLiterals() : + * memcpy() function that won't read beyond more than WILDCOPY_OVERLENGTH bytes past ilimit_w. + * Only called when the sequence ends past ilimit_w, so it only needs to be optimized for single + * large copies. + */ +static void +ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE const* ilimit_w) +{ + assert(iend > ilimit_w); + if (ip <= ilimit_w) { + ZSTD_wildcopy(op, ip, ilimit_w - ip, ZSTD_no_overlap); + op += ilimit_w - ip; + ip = ilimit_w; + } + while (ip < iend) *op++ = *ip++; +} + + +#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1) +#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2) +#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3) +#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */ +#define OFFSET_TO_OFFBASE(o) (assert((o)>0), o + ZSTD_REP_NUM) +#define OFFBASE_IS_OFFSET(o) ((o) > ZSTD_REP_NUM) +#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM) +#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM) +#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */ + +/*! ZSTD_storeSeq() : + * Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t. + * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE(). + * @matchLength : must be >= MINMATCH + * Allowed to over-read literals up to litLimit. +*/ +HINT_INLINE UNUSED_ATTR void +ZSTD_storeSeq(seqStore_t* seqStorePtr, + size_t litLength, const BYTE* literals, const BYTE* litLimit, + U32 offBase, + size_t matchLength) +{ + BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; + BYTE const* const litEnd = literals + litLength; +#if defined(DEBUGLEVEL) && (DEBUGLEVEL >= 6) + static const BYTE* g_start = NULL; + if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ + { U32 const pos = (U32)((const BYTE*)literals - g_start); + DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u", + pos, (U32)litLength, (U32)matchLength, (U32)offBase); + } +#endif + assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); + /* copy Literals */ + assert(seqStorePtr->maxNbLit <= 128 KB); + assert(seqStorePtr->lit + litLength <= seqStorePtr->litStart + seqStorePtr->maxNbLit); + assert(literals + litLength <= litLimit); + if (litEnd <= litLimit_w) { + /* Common case we can use wildcopy. + * First copy 16 bytes, because literals are likely short. + */ + ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(seqStorePtr->lit, literals); + if (litLength > 16) { + ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); + } + } else { + ZSTD_safecopyLiterals(seqStorePtr->lit, literals, litEnd, litLimit_w); + } + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength>0xFFFF) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_literalLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offBase = offBase; + + /* match Length */ + assert(matchLength >= MINMATCH); + { size_t const mlBase = matchLength - MINMATCH; + if (mlBase>0xFFFF) { + assert(seqStorePtr->longLengthType == ZSTD_llt_none); /* there can only be a single long length */ + seqStorePtr->longLengthType = ZSTD_llt_matchLength; + seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + } + seqStorePtr->sequences[0].mlBase = (U16)mlBase; + } + + seqStorePtr->sequences++; +} + +/* ZSTD_updateRep() : + * updates in-place @rep (array of repeat offsets) + * @offBase : sum-type, using numeric representation of ZSTD_storeSeq() + */ +MEM_STATIC void +ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + if (OFFBASE_IS_OFFSET(offBase)) { /* full offset */ + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = OFFBASE_TO_OFFSET(offBase); + } else { /* repcode */ + U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; + if (repCode > 0) { /* note : if repCode==0, no change */ + U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + rep[2] = (repCode >= 2) ? rep[1] : rep[2]; + rep[1] = rep[0]; + rep[0] = currentOffset; + } else { /* repCode == 0 */ + /* nothing to do */ + } + } +} + +typedef struct repcodes_s { + U32 rep[3]; +} repcodes_t; + +MEM_STATIC repcodes_t +ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) +{ + repcodes_t newReps; + ZSTD_memcpy(&newReps, rep, sizeof(newReps)); + ZSTD_updateRep(newReps.rep, offBase, ll0); + return newReps; +} + + +/*-************************************* +* Match length counter +***************************************/ +MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + if (pIn < pInLoopLimit) { + { size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (diff) return ZSTD_NbCommonBytes(diff); } + pIn+=sizeof(size_t); pMatch+=sizeof(size_t); + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } } + if (MEM_64bits() && (pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h, 0); } /* only in zstd_opt.h */ +MEM_STATIC size_t ZSTD_hash3PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash3(MEM_readLE32(ptr), h, s); } + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h, U32 s) { assert(h <= 32); return ((u * prime4bytes) ^ s) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h, 0); } +static size_t ZSTD_hash4PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash4(MEM_readLE32(ptr), h, s); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-40)) * prime5bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash5PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash5(MEM_readLE64(p), h, s); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-48)) * prime6bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash6PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash6(MEM_readLE64(p), h, s); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-56)) * prime7bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash7PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash7(MEM_readLE64(p), h, s); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u) * prime8bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash8PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash8(MEM_readLE64(p), h, s); } + + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtrSalted(const void* p, U32 hBits, U32 mls, const U64 hashSalt) { + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4PtrS(p, hBits, (U32)hashSalt); + case 5: return ZSTD_hash5PtrS(p, hBits, hashSalt); + case 6: return ZSTD_hash6PtrS(p, hBits, hashSalt); + case 7: return ZSTD_hash7PtrS(p, hBits, hashSalt); + case 8: return ZSTD_hash8PtrS(p, hBits, hashSalt); + } +} + + +/** ZSTD_ipow() : + * Return base^exponent. + */ +static U64 ZSTD_ipow(U64 base, U64 exponent) +{ + U64 power = 1; + while (exponent) { + if (exponent & 1) power *= base; + exponent >>= 1; + base *= base; + } + return power; +} + +#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 + +/** ZSTD_rollingHash_append() : + * Add the buffer to the hash value. + */ +static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +{ + BYTE const* istart = (BYTE const*)buf; + size_t pos; + for (pos = 0; pos < size; ++pos) { + hash *= prime8bytes; + hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; + } + return hash; +} + +/** ZSTD_rollingHash_compute() : + * Compute the rolling hash value of the buffer. + */ +MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) +{ + return ZSTD_rollingHash_append(0, buf, size); +} + +/** ZSTD_rollingHash_primePower() : + * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash + * over a window of length bytes. + */ +MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) +{ + return ZSTD_ipow(prime8bytes, length - 1); +} + +/** ZSTD_rollingHash_rotate() : + * Rotate the rolling hash by one byte. + */ +MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) +{ + hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; + hash *= prime8bytes; + hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; + return hash; +} + +/*-************************************* +* Round buffer management +***************************************/ +#if (ZSTD_WINDOWLOG_MAX_64 > 31) +# error "ZSTD_WINDOWLOG_MAX is too large : would overflow ZSTD_CURRENT_MAX" +#endif +/* Max current allowed */ +#define ZSTD_CURRENT_MAX ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)) +/* Maximum chunk size before overflow correction needs to be called again */ +#define ZSTD_CHUNKSIZE_MAX \ + ( ((U32)-1) /* Maximum ending current index */ \ + - ZSTD_CURRENT_MAX) /* Maximum beginning lowLimit */ + +/** + * ZSTD_window_clear(): + * Clears the window containing the history by simply setting it to empty. + */ +MEM_STATIC void ZSTD_window_clear(ZSTD_window_t* window) +{ + size_t const endT = (size_t)(window->nextSrc - window->base); + U32 const end = (U32)endT; + + window->lowLimit = end; + window->dictLimit = end; +} + +MEM_STATIC U32 ZSTD_window_isEmpty(ZSTD_window_t const window) +{ + return window.dictLimit == ZSTD_WINDOW_START_INDEX && + window.lowLimit == ZSTD_WINDOW_START_INDEX && + (window.nextSrc - window.base) == ZSTD_WINDOW_START_INDEX; +} + +/** + * ZSTD_window_hasExtDict(): + * Returns non-zero if the window has a non-empty extDict. + */ +MEM_STATIC U32 ZSTD_window_hasExtDict(ZSTD_window_t const window) +{ + return window.lowLimit < window.dictLimit; +} + +/** + * ZSTD_matchState_dictMode(): + * Inspects the provided matchState and figures out what dictMode should be + * passed to the compressor. + */ +MEM_STATIC ZSTD_dictMode_e ZSTD_matchState_dictMode(const ZSTD_matchState_t *ms) +{ + return ZSTD_window_hasExtDict(ms->window) ? + ZSTD_extDict : + ms->dictMatchState != NULL ? + (ms->dictMatchState->dedicatedDictSearch ? ZSTD_dedicatedDictSearch : ZSTD_dictMatchState) : + ZSTD_noDict; +} + +/* Defining this macro to non-zero tells zstd to run the overflow correction + * code much more frequently. This is very inefficient, and should only be + * used for tests and fuzzers. + */ +#ifndef ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY +# ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 1 +# else +# define ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY 0 +# endif +#endif + +/** + * ZSTD_window_canOverflowCorrect(): + * Returns non-zero if the indices are large enough for overflow correction + * to work correctly without impacting compression ratio. + */ +MEM_STATIC U32 ZSTD_window_canOverflowCorrect(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src) +{ + U32 const cycleSize = 1u << cycleLog; + U32 const curr = (U32)((BYTE const*)src - window.base); + U32 const minIndexToOverflowCorrect = cycleSize + + MAX(maxDist, cycleSize) + + ZSTD_WINDOW_START_INDEX; + + /* Adjust the min index to backoff the overflow correction frequency, + * so we don't waste too much CPU in overflow correction. If this + * computation overflows we don't really care, we just need to make + * sure it is at least minIndexToOverflowCorrect. + */ + U32 const adjustment = window.nbOverflowCorrections + 1; + U32 const adjustedIndex = MAX(minIndexToOverflowCorrect * adjustment, + minIndexToOverflowCorrect); + U32 const indexLargeEnough = curr > adjustedIndex; + + /* Only overflow correct early if the dictionary is invalidated already, + * so we don't hurt compression ratio. + */ + U32 const dictionaryInvalidated = curr > maxDist + loadedDictEnd; + + return indexLargeEnough && dictionaryInvalidated; +} + +/** + * ZSTD_window_needOverflowCorrection(): + * Returns non-zero if the indices are getting too large and need overflow + * protection. + */ +MEM_STATIC U32 ZSTD_window_needOverflowCorrection(ZSTD_window_t const window, + U32 cycleLog, + U32 maxDist, + U32 loadedDictEnd, + void const* src, + void const* srcEnd) +{ + U32 const curr = (U32)((BYTE const*)srcEnd - window.base); + if (ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + if (ZSTD_window_canOverflowCorrect(window, cycleLog, maxDist, loadedDictEnd, src)) { + return 1; + } + } + return curr > ZSTD_CURRENT_MAX; +} + +/** + * ZSTD_window_correctOverflow(): + * Reduces the indices to protect from index overflow. + * Returns the correction made to the indices, which must be applied to every + * stored index. + * + * The least significant cycleLog bits of the indices must remain the same, + * which may be 0. Every index up to maxDist in the past must be valid. + */ +MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, + U32 maxDist, void const* src) +{ + /* preemptive overflow correction: + * 1. correction is large enough: + * lowLimit > (3<<29) ==> current > 3<<29 + 1< (3<<29 + 1< (3<<29) - (1< (3<<29) - (1<<30) (NOTE: chainLog <= 30) + * > 1<<29 + * + * 2. (ip+ZSTD_CHUNKSIZE_MAX - cctx->base) doesn't overflow: + * After correction, current is less than (1<base < 1<<32. + * 3. (cctx->lowLimit + 1< 3<<29 + 1<base); + U32 const currentCycle = curr & cycleMask; + /* Ensure newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX. */ + U32 const currentCycleCorrection = currentCycle < ZSTD_WINDOW_START_INDEX + ? MAX(cycleSize, ZSTD_WINDOW_START_INDEX) + : 0; + U32 const newCurrent = currentCycle + + currentCycleCorrection + + MAX(maxDist, cycleSize); + U32 const correction = curr - newCurrent; + /* maxDist must be a power of two so that: + * (newCurrent & cycleMask) == (curr & cycleMask) + * This is required to not corrupt the chains / binary tree. + */ + assert((maxDist & (maxDist - 1)) == 0); + assert((curr & cycleMask) == (newCurrent & cycleMask)); + assert(curr > newCurrent); + if (!ZSTD_WINDOW_OVERFLOW_CORRECT_FREQUENTLY) { + /* Loose bound, should be around 1<<29 (see above) */ + assert(correction > 1<<28); + } + + window->base += correction; + window->dictBase += correction; + if (window->lowLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->lowLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->lowLimit -= correction; + } + if (window->dictLimit < correction + ZSTD_WINDOW_START_INDEX) { + window->dictLimit = ZSTD_WINDOW_START_INDEX; + } else { + window->dictLimit -= correction; + } + + /* Ensure we can still reference the full window. */ + assert(newCurrent >= maxDist); + assert(newCurrent - maxDist >= ZSTD_WINDOW_START_INDEX); + /* Ensure that lowLimit and dictLimit didn't underflow. */ + assert(window->lowLimit <= newCurrent); + assert(window->dictLimit <= newCurrent); + + ++window->nbOverflowCorrections; + + DEBUGLOG(4, "Correction of 0x%x bytes to lowLimit=0x%x", correction, + window->lowLimit); + return correction; +} + +/** + * ZSTD_window_enforceMaxDist(): + * Updates lowLimit so that: + * (srcEnd - base) - lowLimit == maxDist + loadedDictEnd + * + * It ensures index is valid as long as index >= lowLimit. + * This must be called before a block compression call. + * + * loadedDictEnd is only defined if a dictionary is in use for current compression. + * As the name implies, loadedDictEnd represents the index at end of dictionary. + * The value lies within context's referential, it can be directly compared to blockEndIdx. + * + * If loadedDictEndPtr is NULL, no dictionary is in use, and we use loadedDictEnd == 0. + * If loadedDictEndPtr is not NULL, we set it to zero after updating lowLimit. + * This is because dictionaries are allowed to be referenced fully + * as long as the last byte of the dictionary is in the window. + * Once input has progressed beyond window size, dictionary cannot be referenced anymore. + * + * In normal dict mode, the dictionary lies between lowLimit and dictLimit. + * In dictMatchState mode, lowLimit and dictLimit are the same, + * and the dictionary is below them. + * forceWindow and dictMatchState are therefore incompatible. + */ +MEM_STATIC void +ZSTD_window_enforceMaxDist(ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + + /* - When there is no dictionary : loadedDictEnd == 0. + In which case, the test (blockEndIdx > maxDist) is merely to avoid + overflowing next operation `newLowLimit = blockEndIdx - maxDist`. + - When there is a standard dictionary : + Index referential is copied from the dictionary, + which means it starts from 0. + In which case, loadedDictEnd == dictSize, + and it makes sense to compare `blockEndIdx > maxDist + dictSize` + since `blockEndIdx` also starts from zero. + - When there is an attached dictionary : + loadedDictEnd is expressed within the referential of the context, + so it can be directly compared against blockEndIdx. + */ + if (blockEndIdx > maxDist + loadedDictEnd) { + U32 const newLowLimit = blockEndIdx - maxDist; + if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; + if (window->dictLimit < window->lowLimit) { + DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u", + (unsigned)window->dictLimit, (unsigned)window->lowLimit); + window->dictLimit = window->lowLimit; + } + /* On reaching window size, dictionaries are invalidated */ + if (loadedDictEndPtr) *loadedDictEndPtr = 0; + if (dictMatchStatePtr) *dictMatchStatePtr = NULL; + } +} + +/* Similar to ZSTD_window_enforceMaxDist(), + * but only invalidates dictionary + * when input progresses beyond window size. + * assumption : loadedDictEndPtr and dictMatchStatePtr are valid (non NULL) + * loadedDictEnd uses same referential as window->base + * maxDist is the window size */ +MEM_STATIC void +ZSTD_checkDictValidity(const ZSTD_window_t* window, + const void* blockEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) +{ + assert(loadedDictEndPtr != NULL); + assert(dictMatchStatePtr != NULL); + { U32 const blockEndIdx = (U32)((BYTE const*)blockEnd - window->base); + U32 const loadedDictEnd = *loadedDictEndPtr; + DEBUGLOG(5, "ZSTD_checkDictValidity: blockEndIdx=%u, maxDist=%u, loadedDictEnd=%u", + (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); + assert(blockEndIdx >= loadedDictEnd); + + if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) { + /* On reaching window size, dictionaries are invalidated. + * For simplification, if window size is reached anywhere within next block, + * the dictionary is invalidated for the full block. + * + * We also have to invalidate the dictionary if ZSTD_window_update() has detected + * non-contiguous segments, which means that loadedDictEnd != window->dictLimit. + * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use + * dictMatchState, so setting it to NULL is not a problem. + */ + DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); + *loadedDictEndPtr = 0; + *dictMatchStatePtr = NULL; + } else { + if (*loadedDictEndPtr != 0) { + DEBUGLOG(6, "dictionary considered valid for current block"); + } } } +} + +MEM_STATIC void ZSTD_window_init(ZSTD_window_t* window) { + ZSTD_memset(window, 0, sizeof(*window)); + window->base = (BYTE const*)" "; + window->dictBase = (BYTE const*)" "; + ZSTD_STATIC_ASSERT(ZSTD_DUBT_UNSORTED_MARK < ZSTD_WINDOW_START_INDEX); /* Start above ZSTD_DUBT_UNSORTED_MARK */ + window->dictLimit = ZSTD_WINDOW_START_INDEX; /* start from >0, so that 1st position is valid */ + window->lowLimit = ZSTD_WINDOW_START_INDEX; /* it ensures first and later CCtx usages compress the same */ + window->nextSrc = window->base + ZSTD_WINDOW_START_INDEX; /* see issue #1241 */ + window->nbOverflowCorrections = 0; +} + +/** + * ZSTD_window_update(): + * Updates the window by appending [src, src + srcSize) to the window. + * If it is not contiguous, the current prefix becomes the extDict, and we + * forget about the extDict. Handles overlap of the prefix and extDict. + * Returns non-zero if the segment is contiguous. + */ +MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, + void const* src, size_t srcSize, + int forceNonContiguous) +{ + BYTE const* const ip = (BYTE const*)src; + U32 contiguous = 1; + DEBUGLOG(5, "ZSTD_window_update"); + if (srcSize == 0) + return contiguous; + assert(window->base != NULL); + assert(window->dictBase != NULL); + /* Check if blocks follow each other */ + if (src != window->nextSrc || forceNonContiguous) { + /* not contiguous */ + size_t const distanceFromBase = (size_t)(window->nextSrc - window->base); + DEBUGLOG(5, "Non contiguous blocks, new segment starts at %u", window->dictLimit); + window->lowLimit = window->dictLimit; + assert(distanceFromBase == (size_t)(U32)distanceFromBase); /* should never overflow */ + window->dictLimit = (U32)distanceFromBase; + window->dictBase = window->base; + window->base = ip - distanceFromBase; + /* ms->nextToUpdate = window->dictLimit; */ + if (window->dictLimit - window->lowLimit < HASH_READ_SIZE) window->lowLimit = window->dictLimit; /* too small extDict */ + contiguous = 0; + } + window->nextSrc = ip + srcSize; + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ( (ip+srcSize > window->dictBase + window->lowLimit) + & (ip < window->dictBase + window->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - window->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)window->dictLimit) ? window->dictLimit : (U32)highInputIdx; + window->lowLimit = lowLimitMax; + DEBUGLOG(5, "Overlapping extDict and input : new lowLimit = %u", window->lowLimit); + } + return contiguous; +} + +/** + * Returns the lowest allowed match index. It may either be in the ext-dict or the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestMatchIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.lowLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When using a dictionary the entire dictionary is valid if a single byte of the dictionary + * is within the window. We invalidate the dictionary (and set loadedDictEnd to 0) when it isn't + * valid for the entire block. So this check is sufficient to find the lowest valid match index. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + +/** + * Returns the lowest allowed match index in the prefix. + */ +MEM_STATIC U32 ZSTD_getLowestPrefixIndex(const ZSTD_matchState_t* ms, U32 curr, unsigned windowLog) +{ + U32 const maxDistance = 1U << windowLog; + U32 const lowestValid = ms->window.dictLimit; + U32 const withinWindow = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + U32 const isDictionary = (ms->loadedDictEnd != 0); + /* When computing the lowest prefix index we need to take the dictionary into account to handle + * the edge case where the dictionary and the source are contiguous in memory. + */ + U32 const matchLowest = isDictionary ? lowestValid : withinWindow; + return matchLowest; +} + + + +/* debug functions */ +#if (DEBUGLEVEL>=2) + +MEM_STATIC double ZSTD_fWeight(U32 rawStat) +{ + U32 const fp_accuracy = 8; + U32 const fp_multiplier = (1 << fp_accuracy); + U32 const newStat = rawStat + 1; + U32 const hb = ZSTD_highbit32(newStat); + U32 const BWeight = hb * fp_multiplier; + U32 const FWeight = (newStat << fp_accuracy) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + fp_accuracy < 31); + return (double)weight / fp_multiplier; +} + +/* display a table content, + * listing each element, its frequency, and its predicted bit cost */ +MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) +{ + unsigned u, sum; + for (u=0, sum=0; u<=max; u++) sum += table[u]; + DEBUGLOG(2, "total nb elts: %u", sum); + for (u=0; u<=max; u++) { + DEBUGLOG(2, "%2u: %5u (%.2f)", + u, table[u], ZSTD_fWeight(sum) - ZSTD_fWeight(table[u]) ); + } +} + +#endif + +/* Short Cache */ + +/* Normally, zstd matchfinders follow this flow: + * 1. Compute hash at ip + * 2. Load index from hashTable[hash] + * 3. Check if *ip == *(base + index) + * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss. + * + * Short cache is an optimization which allows us to avoid step 3 most of the time + * when the data doesn't actually match. With short cache, the flow becomes: + * 1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip. + * 2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works. + * 3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue. + * + * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to + * dictMatchState matchfinders. + */ +#define ZSTD_SHORT_CACHE_TAG_BITS 8 +#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1) + +/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable. + * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */ +MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) { + size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK); + assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0); + hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag; +} + +/* Helper function for short cache matchfinders. + * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */ +MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) { + U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK; + U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK; + return tag1 == tag2; +} + +#if defined (__cplusplus) +} +#endif + +/* =============================================================== + * Shared internal declarations + * These prototypes may be called from sources not in lib/compress + * =============================================================== */ + +/* ZSTD_loadCEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * return : size of dictionary header (size of magic number + dict ID + entropy tables) + * assumptions : magic number supposed already checked + * and dictSize >= 8 */ +size_t ZSTD_loadCEntropy(ZSTD_compressedBlockState_t* bs, void* workspace, + const void* const dict, size_t dictSize); + +void ZSTD_reset_compressedBlockState(ZSTD_compressedBlockState_t* bs); + +/* ============================================================== + * Private declarations + * These prototypes shall only be called from within lib/compress + * ============================================================== */ + +/* ZSTD_getCParamsFromCCtxParams() : + * cParams are built depending on compressionLevel, src size hints, + * LDM and manually set compression parameters. + * Note: srcSizeHint == 0 means 0! + */ +ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( + const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); + +/*! ZSTD_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * @return : 0, or an error code */ +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, unsigned long long pledgedSrcSize); + +void ZSTD_resetSeqStore(seqStore_t* ssPtr); + +/*! ZSTD_getCParamsFromCDict() : + * as the name implies */ +ZSTD_compressionParameters ZSTD_getCParamsFromCDict(const ZSTD_CDict* cdict); + +/* ZSTD_compressBegin_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compressBegin_advanced_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictContentType_e dictContentType, + ZSTD_dictTableLoadMethod_e dtlm, + const ZSTD_CDict* cdict, + const ZSTD_CCtx_params* params, + unsigned long long pledgedSrcSize); + +/* ZSTD_compress_advanced_internal() : + * Private use only. To be called from zstdmt_compress.c. */ +size_t ZSTD_compress_advanced_internal(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + const ZSTD_CCtx_params* params); + + +/* ZSTD_writeLastEmptyBlock() : + * output an empty Block with end-of-frame mark to complete a frame + * @return : size of data written into `dst` (== ZSTD_blockHeaderSize (defined in zstd_internal.h)) + * or an error code if `dstCapacity` is too small ( 1 */ +U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); + +/** ZSTD_CCtx_trace() : + * Trace the end of a compression call. + */ +void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); + +/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of + * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. + * Note that the block delimiter must include the last literals of the block. + */ +size_t +ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, + ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); + +/* Returns the number of bytes to move the current read position back by. + * Only non-zero if we ended up splitting a sequence. + * Otherwise, it may return a ZSTD error if something went wrong. + * + * This function will attempt to scan through blockSize bytes + * represented by the sequences in @inSeqs, + * storing any (partial) sequences. + * + * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to + * avoid splitting a match, or to avoid splitting a match such that it would produce a match + * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + */ +size_t +ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); + + +/* =============================================================== + * Deprecated definitions that are still used internally to avoid + * deprecation warnings. These functions are exactly equivalent to + * their public variants, but avoid the deprecation warnings. + * =============================================================== */ + +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +#endif /* ZSTD_COMPRESS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.c new file mode 100644 index 000000000..bfd4f11ab --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.c @@ -0,0 +1,235 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_literals.h" + + +/* ************************************************************** +* Debug Traces +****************************************************************/ +#if DEBUGLEVEL >= 2 + +static size_t showHexa(const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*)src; + size_t u; + for (u=0; u31) + (srcSize>4095); + + DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity); + + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ZSTD_memcpy(ostart + flSize, src, srcSize); + DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); + return srcSize + flSize; +} + +static int allBytesIdentical(const void* src, size_t srcSize) +{ + assert(srcSize >= 1); + assert(src != NULL); + { const BYTE b = ((const BYTE*)src)[0]; + size_t p; + for (p=1; p31) + (srcSize>4095); + + assert(dstCapacity >= 4); (void)dstCapacity; + assert(allBytesIdentical(src, srcSize)); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + default: /* not necessary : flSize is {1,2,3} */ + assert(0); + } + + ostart[flSize] = *(const BYTE*)src; + DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1); + return flSize+1; +} + +/* ZSTD_minLiteralsToCompress() : + * returns minimal amount of literals + * for literal compression to even be attempted. + * Minimum is made tighter as compression strategy increases. + */ +static size_t +ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat) +{ + assert((int)strategy >= 0); + assert((int)strategy <= 9); + /* btultra2 : min 8 bytes; + * then 2x larger for each successive compression strategy + * max threshold 64 bytes */ + { int const shift = MIN(9-(int)strategy, 3); + size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift; + DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc); + return mintc; + } +} + +size_t ZSTD_compressLiterals ( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, + int disableLiteralCompression, + int suspectUncompressible, + int bmi2) +{ + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)", + disableLiteralCompression, (U32)srcSize, dstCapacity); + + DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize)); + + /* Prepare nextEntropy assuming reusing the existing table */ + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + + if (disableLiteralCompression) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + /* if too small, don't even attempt compression (speed opt) */ + if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + + RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); + { HUF_repeat repeat = prevHuf->repeatMode; + int const flags = 0 + | (bmi2 ? HUF_flags_bmi2 : 0) + | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0) + | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0) + | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0); + + typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int); + huf_compress_f huf_compress; + if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; + huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat; + cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize, + src, srcSize, + HUF_SYMBOLVALUE_MAX, LitHufLog, + entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, + &repeat, flags); + DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize); + if (repeat != HUF_repeat_none) { + /* reused the existing table */ + DEBUGLOG(5, "reusing statistics from previous huffman block"); + hType = set_repeat; + } + } + + { size_t const minGain = ZSTD_minGain(srcSize, strategy); + if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } } + if (cLitSize==1) { + /* A return value of 1 signals that the alphabet consists of a single symbol. + * However, in some rare circumstances, it could be the compressed size (a single byte). + * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`. + * (it's also necessary to not generate statistics). + * Therefore, in such a case, actively check that all bytes are identical. */ + if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } } + + if (hType == set_compressed) { + /* using a newly constructed table */ + nextHuf->repeatMode = HUF_repeat_check; + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)srcSize, (U32)(lhSize+cLitSize)); + return lhSize+cLitSize; +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.h new file mode 100644 index 000000000..b060c8ad2 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_literals.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_LITERALS_H +#define ZSTD_COMPRESS_LITERALS_H + +#include "zstd_compress_internal.h" /* ZSTD_hufCTables_t, ZSTD_minGain() */ + + +size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* ZSTD_compressRleLiteralsBlock() : + * Conditions : + * - All bytes in @src are identical + * - dstCapacity >= 4 */ +size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* ZSTD_compressLiterals(): + * @entropyWorkspace: must be aligned on 4-bytes boundaries + * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE + * @suspectUncompressible: sampling checks, to potentially skip huffman coding + */ +size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + int suspectUncompressible, + int bmi2); + +#endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.c new file mode 100644 index 000000000..8872d4d35 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.c @@ -0,0 +1,442 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_sequences.h" + +/** + * -log2(x / 256) lookup table for x in [0, 256). + * If x == 0: Return 0 + * Else: Return floor(-log2(x / 256) * 256) + */ +static unsigned const kInverseProbabilityLog256[256] = { + 0, 2048, 1792, 1642, 1536, 1453, 1386, 1329, 1280, 1236, 1197, 1162, + 1130, 1100, 1073, 1047, 1024, 1001, 980, 960, 941, 923, 906, 889, + 874, 859, 844, 830, 817, 804, 791, 779, 768, 756, 745, 734, + 724, 714, 704, 694, 685, 676, 667, 658, 650, 642, 633, 626, + 618, 610, 603, 595, 588, 581, 574, 567, 561, 554, 548, 542, + 535, 529, 523, 517, 512, 506, 500, 495, 489, 484, 478, 473, + 468, 463, 458, 453, 448, 443, 438, 434, 429, 424, 420, 415, + 411, 407, 402, 398, 394, 390, 386, 382, 377, 373, 370, 366, + 362, 358, 354, 350, 347, 343, 339, 336, 332, 329, 325, 322, + 318, 315, 311, 308, 305, 302, 298, 295, 292, 289, 286, 282, + 279, 276, 273, 270, 267, 264, 261, 258, 256, 253, 250, 247, + 244, 241, 239, 236, 233, 230, 228, 225, 222, 220, 217, 215, + 212, 209, 207, 204, 202, 199, 197, 194, 192, 190, 187, 185, + 182, 180, 178, 175, 173, 171, 168, 166, 164, 162, 159, 157, + 155, 153, 151, 149, 146, 144, 142, 140, 138, 136, 134, 132, + 130, 128, 126, 123, 121, 119, 117, 115, 114, 112, 110, 108, + 106, 104, 102, 100, 98, 96, 94, 93, 91, 89, 87, 85, + 83, 82, 80, 78, 76, 74, 73, 71, 69, 67, 66, 64, + 62, 61, 59, 57, 55, 54, 52, 50, 49, 47, 46, 44, + 42, 41, 39, 37, 36, 34, 33, 31, 30, 28, 26, 25, + 23, 22, 20, 19, 17, 16, 14, 13, 11, 10, 8, 7, + 5, 4, 2, 1, +}; + +static unsigned ZSTD_getFSEMaxSymbolValue(FSE_CTable const* ctable) { + void const* ptr = ctable; + U16 const* u16ptr = (U16 const*)ptr; + U32 const maxSymbolValue = MEM_read16(u16ptr + 1); + return maxSymbolValue; +} + +/** + * Returns true if we should use ncount=-1 else we should + * use ncount=1 for low probability symbols instead. + */ +static unsigned ZSTD_useLowProbCount(size_t const nbSeq) +{ + /* Heuristic: This should cover most blocks <= 16K and + * start to fade out after 16K to about 32K depending on + * compressibility. + */ + return nbSeq >= 2048; +} + +/** + * Returns the cost in bytes of encoding the normalized count header. + * Returns an error if any of the helper functions return an error. + */ +static size_t ZSTD_NCountCost(unsigned const* count, unsigned const max, + size_t const nbSeq, unsigned const FSELog) +{ + BYTE wksp[FSE_NCOUNTBOUND]; + S16 norm[MaxSeq + 1]; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + FORWARD_IF_ERROR(FSE_normalizeCount(norm, tableLog, count, nbSeq, max, ZSTD_useLowProbCount(nbSeq)), ""); + return FSE_writeNCount(wksp, sizeof(wksp), norm, max, tableLog); +} + +/** + * Returns the cost in bits of encoding the distribution described by count + * using the entropy bound. + */ +static size_t ZSTD_entropyCost(unsigned const* count, unsigned const max, size_t const total) +{ + unsigned cost = 0; + unsigned s; + + assert(total > 0); + for (s = 0; s <= max; ++s) { + unsigned norm = (unsigned)((256 * count[s]) / total); + if (count[s] != 0 && norm == 0) + norm = 1; + assert(count[s] < total); + cost += count[s] * kInverseProbabilityLog256[norm]; + } + return cost >> 8; +} + +/** + * Returns the cost in bits of encoding the distribution in count using ctable. + * Returns an error if ctable cannot represent all the symbols in count. + */ +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max) +{ + unsigned const kAccuracyLog = 8; + size_t cost = 0; + unsigned s; + FSE_CState_t cstate; + FSE_initCState(&cstate, ctable); + if (ZSTD_getFSEMaxSymbolValue(ctable) < max) { + DEBUGLOG(5, "Repeat FSE_CTable has maxSymbolValue %u < %u", + ZSTD_getFSEMaxSymbolValue(ctable), max); + return ERROR(GENERIC); + } + for (s = 0; s <= max; ++s) { + unsigned const tableLog = cstate.stateLog; + unsigned const badCost = (tableLog + 1) << kAccuracyLog; + unsigned const bitCost = FSE_bitCost(cstate.symbolTT, tableLog, s, kAccuracyLog); + if (count[s] == 0) + continue; + if (bitCost >= badCost) { + DEBUGLOG(5, "Repeat FSE_CTable has Prob[%u] == 0", s); + return ERROR(GENERIC); + } + cost += (size_t)count[s] * bitCost; + } + return cost >> kAccuracyLog; +} + +/** + * Returns the cost in bits of encoding the distribution in count using the + * table described by norm. The max symbol support by norm is assumed >= max. + * norm must be valid for every symbol with non-zero probability in count. + */ +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max) +{ + unsigned const shift = 8 - accuracyLog; + size_t cost = 0; + unsigned s; + assert(accuracyLog <= 8); + for (s = 0; s <= max; ++s) { + unsigned const normAcc = (norm[s] != -1) ? (unsigned)norm[s] : 1; + unsigned const norm256 = normAcc << shift; + assert(norm256 > 0); + assert(norm256 < 256); + cost += count[s] * kInverseProbabilityLog256[norm256]; + } + return cost >> 8; +} + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy) +{ + ZSTD_STATIC_ASSERT(ZSTD_defaultDisallowed == 0 && ZSTD_defaultAllowed != 0); + if (mostFrequent == nbSeq) { + *repeatMode = FSE_repeat_none; + if (isDefaultAllowed && nbSeq <= 2) { + /* Prefer set_basic over set_rle when there are 2 or fewer symbols, + * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. + * If basic encoding isn't possible, always choose RLE. + */ + DEBUGLOG(5, "Selected set_basic"); + return set_basic; + } + DEBUGLOG(5, "Selected set_rle"); + return set_rle; + } + if (strategy < ZSTD_lazy) { + if (isDefaultAllowed) { + size_t const staticFse_nbSeq_max = 1000; + size_t const mult = 10 - strategy; + size_t const baseLog = 3; + size_t const dynamicFse_nbSeq_min = (((size_t)1 << defaultNormLog) * mult) >> baseLog; /* 28-36 for offset, 56-72 for lengths */ + assert(defaultNormLog >= 5 && defaultNormLog <= 6); /* xx_DEFAULTNORMLOG */ + assert(mult <= 9 && mult >= 7); + if ( (*repeatMode == FSE_repeat_valid) + && (nbSeq < staticFse_nbSeq_max) ) { + DEBUGLOG(5, "Selected set_repeat"); + return set_repeat; + } + if ( (nbSeq < dynamicFse_nbSeq_min) + || (mostFrequent < (nbSeq >> (defaultNormLog-1))) ) { + DEBUGLOG(5, "Selected set_basic"); + /* The format allows default tables to be repeated, but it isn't useful. + * When using simple heuristics to select encoding type, we don't want + * to confuse these tables with dictionaries. When running more careful + * analysis, we don't need to waste time checking both repeating tables + * and default tables. + */ + *repeatMode = FSE_repeat_none; + return set_basic; + } + } + } else { + size_t const basicCost = isDefaultAllowed ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, count, max) : ERROR(GENERIC); + size_t const repeatCost = *repeatMode != FSE_repeat_none ? ZSTD_fseBitCost(prevCTable, count, max) : ERROR(GENERIC); + size_t const NCountCost = ZSTD_NCountCost(count, max, nbSeq, FSELog); + size_t const compressedCost = (NCountCost << 3) + ZSTD_entropyCost(count, max, nbSeq); + + if (isDefaultAllowed) { + assert(!ZSTD_isError(basicCost)); + assert(!(*repeatMode == FSE_repeat_valid && ZSTD_isError(repeatCost))); + } + assert(!ZSTD_isError(NCountCost)); + assert(compressedCost < ERROR(maxCode)); + DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); + if (basicCost <= repeatCost && basicCost <= compressedCost) { + DEBUGLOG(5, "Selected set_basic"); + assert(isDefaultAllowed); + *repeatMode = FSE_repeat_none; + return set_basic; + } + if (repeatCost <= compressedCost) { + DEBUGLOG(5, "Selected set_repeat"); + assert(!ZSTD_isError(repeatCost)); + return set_repeat; + } + assert(compressedCost < basicCost && compressedCost < repeatCost); + } + DEBUGLOG(5, "Selected set_compressed"); + *repeatMode = FSE_repeat_check; + return set_compressed; +} + +typedef struct { + S16 norm[MaxSeq + 1]; + U32 wksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, MaxFSELog)]; +} ZSTD_BuildCTableWksp; + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize) +{ + BYTE* op = (BYTE*)dst; + const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); + + switch (type) { + case set_rle: + FORWARD_IF_ERROR(FSE_buildCTable_rle(nextCTable, (BYTE)max), ""); + RETURN_ERROR_IF(dstCapacity==0, dstSize_tooSmall, "not enough space"); + *op = codeTable[0]; + return 1; + case set_repeat: + ZSTD_memcpy(nextCTable, prevCTable, prevCTableSize); + return 0; + case set_basic: + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, defaultNorm, defaultMax, defaultNormLog, entropyWorkspace, entropyWorkspaceSize), ""); /* note : could be pre-calculated */ + return 0; + case set_compressed: { + ZSTD_BuildCTableWksp* wksp = (ZSTD_BuildCTableWksp*)entropyWorkspace; + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(FSELog, nbSeq, max); + if (count[codeTable[nbSeq-1]] > 1) { + count[codeTable[nbSeq-1]]--; + nbSeq_1--; + } + assert(nbSeq_1 > 1); + assert(entropyWorkspaceSize >= sizeof(ZSTD_BuildCTableWksp)); + (void)entropyWorkspaceSize; + FORWARD_IF_ERROR(FSE_normalizeCount(wksp->norm, tableLog, count, nbSeq_1, max, ZSTD_useLowProbCount(nbSeq_1)), "FSE_normalizeCount failed"); + assert(oend >= op); + { size_t const NCountSize = FSE_writeNCount(op, (size_t)(oend - op), wksp->norm, max, tableLog); /* overflow protected */ + FORWARD_IF_ERROR(NCountSize, "FSE_writeNCount failed"); + FORWARD_IF_ERROR(FSE_buildCTable_wksp(nextCTable, wksp->norm, max, tableLog, wksp->wksp, sizeof(wksp->wksp)), "FSE_buildCTable_wksp failed"); + return NCountSize; + } + } + default: assert(0); RETURN_ERROR(GENERIC, "impossible to reach"); + } +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_encodeSequences_body( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, dst, dstCapacity)), + dstSize_tooSmall, "not enough space remaining"); + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].mlBase, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + if (longOffsets) { + U32 const ofBits = ofCodeTable[nbSeq-1]; + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, extraBits); + BIT_flushBits(&blockStream); + } + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase >> extraBits, + ofBits - extraBits); + } else { + BIT_addBits(&blockStream, sequences[nbSeq-1].offBase, ofCodeTable[nbSeq-1]); + } + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].mlBase, mlBits); + if (MEM_32bits() || (ofBits+mlBits+llBits > 56)) BIT_flushBits(&blockStream); + if (longOffsets) { + unsigned const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN-1); + if (extraBits) { + BIT_addBits(&blockStream, sequences[n].offBase, extraBits); + BIT_flushBits(&blockStream); /* (7)*/ + } + BIT_addBits(&blockStream, sequences[n].offBase >> extraBits, + ofBits - extraBits); /* 31 */ + } else { + BIT_addBits(&blockStream, sequences[n].offBase, ofBits); /* 31 */ + } + BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); + } } + + DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); + FSE_flushCState(&blockStream, &stateMatchLength); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing Off state with %u bits", stateOffsetBits.stateLog); + FSE_flushCState(&blockStream, &stateOffsetBits); + DEBUGLOG(6, "ZSTD_encodeSequences: flushing LL state with %u bits", stateLitLength.stateLog); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + RETURN_ERROR_IF(streamSize==0, dstSize_tooSmall, "not enough space"); + return streamSize; + } +} + +static size_t +ZSTD_encodeSequences_default( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + + +#if DYNAMIC_BMI2 + +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_encodeSequences_bmi2( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets) +{ + return ZSTD_encodeSequences_body(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} + +#endif + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) +{ + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); +#if DYNAMIC_BMI2 + if (bmi2) { + return ZSTD_encodeSequences_bmi2(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); + } +#endif + (void)bmi2; + return ZSTD_encodeSequences_default(dst, dstCapacity, + CTable_MatchLength, mlCodeTable, + CTable_OffsetBits, ofCodeTable, + CTable_LitLength, llCodeTable, + sequences, nbSeq, longOffsets); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.h new file mode 100644 index 000000000..4a3a05da9 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_sequences.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_SEQUENCES_H +#define ZSTD_COMPRESS_SEQUENCES_H + +#include "../common/fse.h" /* FSE_repeat, FSE_CTable */ +#include "../common/zstd_internal.h" /* symbolEncodingType_e, ZSTD_strategy */ + +typedef enum { + ZSTD_defaultDisallowed = 0, + ZSTD_defaultAllowed = 1 +} ZSTD_defaultPolicy_e; + +symbolEncodingType_e +ZSTD_selectEncodingType( + FSE_repeat* repeatMode, unsigned const* count, unsigned const max, + size_t const mostFrequent, size_t nbSeq, unsigned const FSELog, + FSE_CTable const* prevCTable, + short const* defaultNorm, U32 defaultNormLog, + ZSTD_defaultPolicy_e const isDefaultAllowed, + ZSTD_strategy const strategy); + +size_t +ZSTD_buildCTable(void* dst, size_t dstCapacity, + FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, + unsigned* count, U32 max, + const BYTE* codeTable, size_t nbSeq, + const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, + const FSE_CTable* prevCTable, size_t prevCTableSize, + void* entropyWorkspace, size_t entropyWorkspaceSize); + +size_t ZSTD_encodeSequences( + void* dst, size_t dstCapacity, + FSE_CTable const* CTable_MatchLength, BYTE const* mlCodeTable, + FSE_CTable const* CTable_OffsetBits, BYTE const* ofCodeTable, + FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, + seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2); + +size_t ZSTD_fseBitCost( + FSE_CTable const* ctable, + unsigned const* count, + unsigned const max); + +size_t ZSTD_crossEntropyCost(short const* norm, unsigned accuracyLog, + unsigned const* count, unsigned const max); +#endif /* ZSTD_COMPRESS_SEQUENCES_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.c new file mode 100644 index 000000000..638c4acbe --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.c @@ -0,0 +1,577 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + /*-************************************* + * Dependencies + ***************************************/ +#include "zstd_compress_superblock.h" + +#include "../common/zstd_internal.h" /* ZSTD_getSequenceLength */ +#include "hist.h" /* HIST_countFast_wksp */ +#include "zstd_compress_internal.h" /* ZSTD_[huf|fse|entropy]CTablesMetadata_t */ +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" + +/** ZSTD_compressSubBlock_literal() : + * Compresses literals section for a sub-block. + * When we have to write the Huffman table we will sometimes choose a header + * size larger than necessary. This is because we have to pick the header size + * before we know the table size + compressed size, so we have a bound on the + * table size. If we guessed incorrectly, we fall back to uncompressed literals. + * + * We write the header when writeEntropy=1 and set entropyWritten=1 when we succeeded + * in writing the header, otherwise it is set to 0. + * + * hufMetadata->hType has literals block type info. + * If it is set_basic, all sub-blocks literals section will be Raw_Literals_Block. + * If it is set_rle, all sub-blocks literals section will be RLE_Literals_Block. + * If it is set_compressed, first sub-block's literals section will be Compressed_Literals_Block + * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block + * and the following sub-blocks' literals sections will be Treeless_Literals_Block. + * @return : compressed size of literals section of a sub-block + * Or 0 if unable to compress. + * Or error code */ +static size_t +ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + size_t const header = writeEntropy ? 200 : 0; + size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart + lhSize; + U32 const singleStream = lhSize == 3; + symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; + size_t cLitSize = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); + + *entropyWritten = 0; + if (litSize == 0 || hufMetadata->hType == set_basic) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } else if (hufMetadata->hType == set_rle) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using rle literal"); + return ZSTD_compressRleLiteralsBlock(dst, dstSize, literals, litSize); + } + + assert(litSize > 0); + assert(hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat); + + if (writeEntropy && hufMetadata->hType == set_compressed) { + ZSTD_memcpy(op, hufMetadata->hufDesBuffer, hufMetadata->hufDesSize); + op += hufMetadata->hufDesSize; + cLitSize += hufMetadata->hufDesSize; + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); + } + + { int const flags = bmi2 ? HUF_flags_bmi2 : 0; + const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags) + : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags); + op += cSize; + cLitSize += cSize; + if (cSize == 0 || ERR_isError(cSize)) { + DEBUGLOG(5, "Failed to write entropy tables %s", ZSTD_getErrorName(cSize)); + return 0; + } + /* If we expand and we aren't writing a header then emit uncompressed */ + if (!writeEntropy && cLitSize >= litSize) { + DEBUGLOG(5, "ZSTD_compressSubBlock_literal using raw literal because uncompressible"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + /* If we are writing headers then allow expansion that doesn't change our header size. */ + if (lhSize < (size_t)(3 + (cLitSize >= 1 KB) + (cLitSize >= 16 KB))) { + assert(cLitSize > litSize); + DEBUGLOG(5, "Literals expanded beyond allowed header size"); + return ZSTD_noCompressLiterals(dst, dstSize, literals, litSize); + } + DEBUGLOG(5, "ZSTD_compressSubBlock_literal (cSize=%zu)", cSize); + } + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)litSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + default: /* not possible : lhSize is {3,4,5} */ + assert(0); + } + *entropyWritten = 1; + DEBUGLOG(5, "Compressed literals: %u -> %u", (U32)litSize, (U32)(op-ostart)); + return op-ostart; +} + +static size_t +ZSTD_seqDecompressedSize(seqStore_t const* seqStore, + const seqDef* sequences, size_t nbSeq, + size_t litSize, int lastSequence) +{ + const seqDef* const sstart = sequences; + const seqDef* const send = sequences + nbSeq; + const seqDef* sp = sstart; + size_t matchLengthSum = 0; + size_t litLengthSum = 0; + (void)(litLengthSum); /* suppress unused variable warning on some environments */ + while (send-sp > 0) { + ZSTD_sequenceLength const seqLen = ZSTD_getSequenceLength(seqStore, sp); + litLengthSum += seqLen.litLength; + matchLengthSum += seqLen.matchLength; + sp++; + } + assert(litLengthSum <= litSize); + if (!lastSequence) { + assert(litLengthSum == litSize); + } + return matchLengthSum + litSize; +} + +/** ZSTD_compressSubBlock_sequences() : + * Compresses sequences section for a sub-block. + * fseMetadata->llType, fseMetadata->ofType, and fseMetadata->mlType have + * symbol compression modes for the super-block. + * The first successfully compressed block will have these in its header. + * We set entropyWritten=1 when we succeed in compressing the sequences. + * The following sub-blocks will always have repeat mode. + * @return : compressed size of sequences section of a sub-block + * Or 0 if it is unable to compress + * Or error code. */ +static size_t +ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, int writeEntropy, int* entropyWritten) +{ + const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + BYTE* seqHead; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (nbSeq=%zu, writeEntropy=%d, longOffsets=%d)", nbSeq, writeEntropy, longOffsets); + + *entropyWritten = 0; + /* Sequences Header */ + RETURN_ERROR_IF((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead*/, + dstSize_tooSmall, ""); + if (nbSeq < 0x7F) + *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) + op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else + op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) { + return op - ostart; + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (seqHeadSize=%u)", (unsigned)(op-ostart)); + + if (writeEntropy) { + const U32 LLtype = fseMetadata->llType; + const U32 Offtype = fseMetadata->ofType; + const U32 MLtype = fseMetadata->mlType; + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (fseTablesSize=%zu)", fseMetadata->fseTablesSize); + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + ZSTD_memcpy(op, fseMetadata->fseTablesBuffer, fseMetadata->fseTablesSize); + op += fseMetadata->fseTablesSize; + } else { + const U32 repeat = set_repeat; + *seqHead = (BYTE)((repeat<<6) + (repeat<<4) + (repeat<<2)); + } + + { size_t const bitstreamSize = ZSTD_encodeSequences( + op, oend - op, + fseTables->matchlengthCTable, mlCode, + fseTables->offcodeCTable, ofCode, + fseTables->litlengthCTable, llCode, + sequences, nbSeq, + longOffsets, bmi2); + FORWARD_IF_ERROR(bitstreamSize, "ZSTD_encodeSequences failed"); + op += bitstreamSize; + /* zstd versions <= 1.3.4 mistakenly report corruption when + * FSE_readNCount() receives a buffer < 4 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1146. + * This can happen when the last set_compressed table present is 2 + * bytes and the bitstream is only one byte. + * In this exceedingly rare case, we will simply emit an uncompressed + * block, since it isn't worth optimizing. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (writeEntropy && fseMetadata->lastCountSize && fseMetadata->lastCountSize + bitstreamSize < 4) { + /* NCountSize >= 2 && bitstreamSize > 0 ==> lastCountSize == 3 */ + assert(fseMetadata->lastCountSize + bitstreamSize == 3); + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.3.4 by " + "emitting an uncompressed block."); + return 0; + } +#endif + DEBUGLOG(5, "ZSTD_compressSubBlock_sequences (bitstreamSize=%zu)", bitstreamSize); + } + + /* zstd versions <= 1.4.0 mistakenly report error when + * sequences section body size is less than 3 bytes. + * Fixed by https://github.com/facebook/zstd/pull/1664. + * This can happen when the previous sequences section block is compressed + * with rle mode and the current block's sequences section is compressed + * with repeat mode where sequences section body size can be 1 byte. + */ +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + if (op-seqHead < 4) { + DEBUGLOG(5, "Avoiding bug in zstd decoder in versions <= 1.4.0 by emitting " + "an uncompressed block when sequences are < 4 bytes"); + return 0; + } +#endif + + *entropyWritten = 1; + return op - ostart; +} + +/** ZSTD_compressSubBlock() : + * Compresses a single sub-block. + * @return : compressed size of the sub-block + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock(const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* literals, size_t litSize, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, + int writeLitEntropy, int writeSeqEntropy, + int* litEntropyWritten, int* seqEntropyWritten, + U32 lastBlock) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart + ZSTD_blockHeaderSize; + DEBUGLOG(5, "ZSTD_compressSubBlock (litSize=%zu, nbSeq=%zu, writeLitEntropy=%d, writeSeqEntropy=%d, lastBlock=%d)", + litSize, nbSeq, writeLitEntropy, writeSeqEntropy, lastBlock); + { size_t cLitSize = ZSTD_compressSubBlock_literal((const HUF_CElt*)entropy->huf.CTable, + &entropyMetadata->hufMetadata, literals, litSize, + op, oend-op, bmi2, writeLitEntropy, litEntropyWritten); + FORWARD_IF_ERROR(cLitSize, "ZSTD_compressSubBlock_literal failed"); + if (cLitSize == 0) return 0; + op += cLitSize; + } + { size_t cSeqSize = ZSTD_compressSubBlock_sequences(&entropy->fse, + &entropyMetadata->fseMetadata, + sequences, nbSeq, + llCode, mlCode, ofCode, + cctxParams, + op, oend-op, + bmi2, writeSeqEntropy, seqEntropyWritten); + FORWARD_IF_ERROR(cSeqSize, "ZSTD_compressSubBlock_sequences failed"); + if (cSeqSize == 0) return 0; + op += cSeqSize; + } + /* Write block header */ + { size_t cSize = (op-ostart)-ZSTD_blockHeaderSize; + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(ostart, cBlockHeader24); + } + return op-ostart; +} + +static size_t ZSTD_estimateSubBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + unsigned* const countWksp = (unsigned*)workspace; + unsigned maxSymbolValue = 255; + size_t literalSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + + if (hufMetadata->hType == set_basic) return litSize; + else if (hufMetadata->hType == set_rle) return 1; + else if (hufMetadata->hType == set_compressed || hufMetadata->hType == set_repeat) { + size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)literals, litSize, workspace, wkspSize); + if (ZSTD_isError(largest)) return litSize; + { size_t cLitSizeEstimate = HUF_estimateCompressedSize((const HUF_CElt*)huf->CTable, countWksp, maxSymbolValue); + if (writeEntropy) cLitSizeEstimate += hufMetadata->hufDesSize; + return cLitSizeEstimate + literalSectionHeaderSize; + } } + assert(0); /* impossible */ + return 0; +} + +static size_t ZSTD_estimateSubBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, unsigned maxCode, + size_t nbSeq, const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) +{ + unsigned* const countWksp = (unsigned*)workspace; + const BYTE* ctp = codeTable; + const BYTE* const ctStart = ctp; + const BYTE* const ctEnd = ctStart + nbSeq; + size_t cSymbolTypeSizeEstimateInBits = 0; + unsigned max = maxCode; + + HIST_countFast_wksp(countWksp, &max, codeTable, nbSeq, workspace, wkspSize); /* can't fail */ + if (type == set_basic) { + /* We selected this encoding type, so it must be valid. */ + assert(max <= defaultMax); + cSymbolTypeSizeEstimateInBits = max <= defaultMax + ? ZSTD_crossEntropyCost(defaultNorm, defaultNormLog, countWksp, max) + : ERROR(GENERIC); + } else if (type == set_rle) { + cSymbolTypeSizeEstimateInBits = 0; + } else if (type == set_compressed || type == set_repeat) { + cSymbolTypeSizeEstimateInBits = ZSTD_fseBitCost(fseCTable, countWksp, max); + } + if (ZSTD_isError(cSymbolTypeSizeEstimateInBits)) return nbSeq * 10; + while (ctp < ctEnd) { + if (additionalBits) cSymbolTypeSizeEstimateInBits += additionalBits[*ctp]; + else cSymbolTypeSizeEstimateInBits += *ctp; /* for offset, offset code is also the number of additional bits */ + ctp++; + } + return cSymbolTypeSizeEstimateInBits / 8; +} + +static size_t ZSTD_estimateSubBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) +{ + size_t const sequencesSectionHeaderSize = 3; /* Use hard coded size of 3 bytes */ + size_t cSeqSizeEstimate = 0; + if (nbSeq == 0) return sequencesSectionHeaderSize; + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, MaxOff, + nbSeq, fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->llType, llCodeTable, MaxLL, + nbSeq, fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); + cSeqSizeEstimate += ZSTD_estimateSubBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, MaxML, + nbSeq, fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); + if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; + return cSeqSizeEstimate + sequencesSectionHeaderSize; +} + +static size_t ZSTD_estimateSubBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) { + size_t cSizeEstimate = 0; + cSizeEstimate += ZSTD_estimateSubBlockSize_literal(literals, litSize, + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); + cSizeEstimate += ZSTD_estimateSubBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); + return cSizeEstimate + ZSTD_blockHeaderSize; +} + +static int ZSTD_needSequenceEntropyTables(ZSTD_fseCTablesMetadata_t const* fseMetadata) +{ + if (fseMetadata->llType == set_compressed || fseMetadata->llType == set_rle) + return 1; + if (fseMetadata->mlType == set_compressed || fseMetadata->mlType == set_rle) + return 1; + if (fseMetadata->ofType == set_compressed || fseMetadata->ofType == set_rle) + return 1; + return 0; +} + +/** ZSTD_compressSubBlock_multi() : + * Breaks super-block into multiple sub-blocks and compresses them. + * Entropy will be written to the first block. + * The following blocks will use repeat mode to compress. + * All sub-blocks are compressed blocks (no raw or rle blocks). + * @return : compressed size of the super block (which is multiple ZSTD blocks) + * Or 0 if it failed to compress. */ +static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, + const ZSTD_compressedBlockState_t* prevCBlock, + ZSTD_compressedBlockState_t* nextCBlock, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const int bmi2, U32 lastBlock, + void* workspace, size_t wkspSize) +{ + const seqDef* const sstart = seqStorePtr->sequencesStart; + const seqDef* const send = seqStorePtr->sequences; + const seqDef* sp = sstart; + const BYTE* const lstart = seqStorePtr->litStart; + const BYTE* const lend = seqStorePtr->lit; + const BYTE* lp = lstart; + BYTE const* ip = (BYTE const*)src; + BYTE const* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + const BYTE* llCodePtr = seqStorePtr->llCode; + const BYTE* mlCodePtr = seqStorePtr->mlCode; + const BYTE* ofCodePtr = seqStorePtr->ofCode; + size_t targetCBlockSize = cctxParams->targetCBlockSize; + size_t litSize, seqCount; + int writeLitEntropy = entropyMetadata->hufMetadata.hType == set_compressed; + int writeSeqEntropy = 1; + int lastSequence = 0; + + DEBUGLOG(5, "ZSTD_compressSubBlock_multi (litSize=%u, nbSeq=%u)", + (unsigned)(lend-lp), (unsigned)(send-sstart)); + + litSize = 0; + seqCount = 0; + do { + size_t cBlockSizeEstimate = 0; + if (sstart == send) { + lastSequence = 1; + } else { + const seqDef* const sequence = sp + seqCount; + lastSequence = sequence == send - 1; + litSize += ZSTD_getSequenceLength(seqStorePtr, sequence).litLength; + seqCount++; + } + if (lastSequence) { + assert(lp <= lend); + assert(litSize <= (size_t)(lend - lp)); + litSize = (size_t)(lend - lp); + } + /* I think there is an optimization opportunity here. + * Calling ZSTD_estimateSubBlockSize for every sequence can be wasteful + * since it recalculates estimate from scratch. + * For example, it would recount literal distribution and symbol codes every time. + */ + cBlockSizeEstimate = ZSTD_estimateSubBlockSize(lp, litSize, ofCodePtr, llCodePtr, mlCodePtr, seqCount, + &nextCBlock->entropy, entropyMetadata, + workspace, wkspSize, writeLitEntropy, writeSeqEntropy); + if (cBlockSizeEstimate > targetCBlockSize || lastSequence) { + int litEntropyWritten = 0; + int seqEntropyWritten = 0; + const size_t decompressedSize = ZSTD_seqDecompressedSize(seqStorePtr, sp, seqCount, litSize, lastSequence); + const size_t cSize = ZSTD_compressSubBlock(&nextCBlock->entropy, entropyMetadata, + sp, seqCount, + lp, litSize, + llCodePtr, mlCodePtr, ofCodePtr, + cctxParams, + op, oend-op, + bmi2, writeLitEntropy, writeSeqEntropy, + &litEntropyWritten, &seqEntropyWritten, + lastBlock && lastSequence); + FORWARD_IF_ERROR(cSize, "ZSTD_compressSubBlock failed"); + if (cSize > 0 && cSize < decompressedSize) { + DEBUGLOG(5, "Committed the sub-block"); + assert(ip + decompressedSize <= iend); + ip += decompressedSize; + sp += seqCount; + lp += litSize; + op += cSize; + llCodePtr += seqCount; + mlCodePtr += seqCount; + ofCodePtr += seqCount; + litSize = 0; + seqCount = 0; + /* Entropy only needs to be written once */ + if (litEntropyWritten) { + writeLitEntropy = 0; + } + if (seqEntropyWritten) { + writeSeqEntropy = 0; + } + } + } + } while (!lastSequence); + if (writeLitEntropy) { + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has literal entropy tables unwritten"); + ZSTD_memcpy(&nextCBlock->entropy.huf, &prevCBlock->entropy.huf, sizeof(prevCBlock->entropy.huf)); + } + if (writeSeqEntropy && ZSTD_needSequenceEntropyTables(&entropyMetadata->fseMetadata)) { + /* If we haven't written our entropy tables, then we've violated our contract and + * must emit an uncompressed block. + */ + DEBUGLOG(5, "ZSTD_compressSubBlock_multi has sequence entropy tables unwritten"); + return 0; + } + if (ip < iend) { + size_t const cSize = ZSTD_noCompressBlock(op, oend - op, ip, iend - ip, lastBlock); + DEBUGLOG(5, "ZSTD_compressSubBlock_multi last sub-block uncompressed, %zu bytes", (size_t)(iend - ip)); + FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); + assert(cSize != 0); + op += cSize; + /* We have to regenerate the repcodes because we've skipped some sequences */ + if (sp < send) { + seqDef const* seq; + repcodes_t rep; + ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); + for (seq = sstart; seq < sp; ++seq) { + ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + } + ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); + } + } + DEBUGLOG(5, "ZSTD_compressSubBlock_multi compressed"); + return op-ostart; +} + +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock) { + ZSTD_entropyCTablesMetadata_t entropyMetadata; + + FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(&zc->seqStore, + &zc->blockState.prevCBlock->entropy, + &zc->blockState.nextCBlock->entropy, + &zc->appliedParams, + &entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); + + return ZSTD_compressSubBlock_multi(&zc->seqStore, + zc->blockState.prevCBlock, + zc->blockState.nextCBlock, + &entropyMetadata, + &zc->appliedParams, + dst, dstCapacity, + src, srcSize, + zc->bmi2, lastBlock, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.h new file mode 100644 index 000000000..8e494f0d5 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_compress_superblock.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPRESS_ADVANCED_H +#define ZSTD_COMPRESS_ADVANCED_H + +/*-************************************* +* Dependencies +***************************************/ + +#include "../zstd.h" /* ZSTD_CCtx */ + +/*-************************************* +* Target Compressed Block Size +***************************************/ + +/* ZSTD_compressSuperBlock() : + * Used to compress a super block when targetCBlockSize is being used. + * The given block will be compressed into multiple sub blocks that are around targetCBlockSize. */ +size_t ZSTD_compressSuperBlock(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + unsigned lastBlock); + +#endif /* ZSTD_COMPRESS_ADVANCED_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h new file mode 100644 index 000000000..cc7fb1c71 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_cwksp.h @@ -0,0 +1,742 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CWKSP_H +#define ZSTD_CWKSP_H + +/*-************************************* +* Dependencies +***************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "../common/zstd_internal.h" +#include "../common/portability_macros.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-************************************* +* Constants +***************************************/ + +/* Since the workspace is effectively its own little malloc implementation / + * arena, when we run under ASAN, we should similarly insert redzones between + * each internal element of the workspace, so ASAN will catch overruns that + * reach outside an object but that stay inside the workspace. + * + * This defines the size of that redzone. + */ +#ifndef ZSTD_CWKSP_ASAN_REDZONE_SIZE +#define ZSTD_CWKSP_ASAN_REDZONE_SIZE 128 +#endif + + +/* Set our tables and aligneds to align by 64 bytes */ +#define ZSTD_CWKSP_ALIGNMENT_BYTES 64 + +/*-************************************* +* Structures +***************************************/ +typedef enum { + ZSTD_cwksp_alloc_objects, + ZSTD_cwksp_alloc_aligned_init_once, + ZSTD_cwksp_alloc_aligned, + ZSTD_cwksp_alloc_buffers +} ZSTD_cwksp_alloc_phase_e; + +/** + * Used to describe whether the workspace is statically allocated (and will not + * necessarily ever be freed), or if it's dynamically allocated and we can + * expect a well-formed caller to free this. + */ +typedef enum { + ZSTD_cwksp_dynamic_alloc, + ZSTD_cwksp_static_alloc +} ZSTD_cwksp_static_alloc_e; + +/** + * Zstd fits all its internal datastructures into a single continuous buffer, + * so that it only needs to perform a single OS allocation (or so that a buffer + * can be provided to it and it can perform no allocations at all). This buffer + * is called the workspace. + * + * Several optimizations complicate that process of allocating memory ranges + * from this workspace for each internal datastructure: + * + * - These different internal datastructures have different setup requirements: + * + * - The static objects need to be cleared once and can then be trivially + * reused for each compression. + * + * - Various buffers don't need to be initialized at all--they are always + * written into before they're read. + * + * - The matchstate tables have a unique requirement that they don't need + * their memory to be totally cleared, but they do need the memory to have + * some bound, i.e., a guarantee that all values in the memory they've been + * allocated is less than some maximum value (which is the starting value + * for the indices that they will then use for compression). When this + * guarantee is provided to them, they can use the memory without any setup + * work. When it can't, they have to clear the area. + * + * - These buffers also have different alignment requirements. + * + * - We would like to reuse the objects in the workspace for multiple + * compressions without having to perform any expensive reallocation or + * reinitialization work. + * + * - We would like to be able to efficiently reuse the workspace across + * multiple compressions **even when the compression parameters change** and + * we need to resize some of the objects (where possible). + * + * To attempt to manage this buffer, given these constraints, the ZSTD_cwksp + * abstraction was created. It works as follows: + * + * Workspace Layout: + * + * [ ... workspace ... ] + * [objects][tables ->] free space [<- buffers][<- aligned][<- init once] + * + * The various objects that live in the workspace are divided into the + * following categories, and are allocated separately: + * + * - Static objects: this is optionally the enclosing ZSTD_CCtx or ZSTD_CDict, + * so that literally everything fits in a single buffer. Note: if present, + * this must be the first object in the workspace, since ZSTD_customFree{CCtx, + * CDict}() rely on a pointer comparison to see whether one or two frees are + * required. + * + * - Fixed size objects: these are fixed-size, fixed-count objects that are + * nonetheless "dynamically" allocated in the workspace so that we can + * control how they're initialized separately from the broader ZSTD_CCtx. + * Examples: + * - Entropy Workspace + * - 2 x ZSTD_compressedBlockState_t + * - CDict dictionary contents + * + * - Tables: these are any of several different datastructures (hash tables, + * chain tables, binary trees) that all respect a common format: they are + * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). + * Their sizes depend on the cparams. These tables are 64-byte aligned. + * + * - Init once: these buffers require to be initialized at least once before + * use. They should be used when we want to skip memory initialization + * while not triggering memory checkers (like Valgrind) when reading from + * from this memory without writing to it first. + * These buffers should be used carefully as they might contain data + * from previous compressions. + * Buffers are aligned to 64 bytes. + * + * - Aligned: these buffers don't require any initialization before they're + * used. The user of the buffer should make sure they write into a buffer + * location before reading from it. + * Buffers are aligned to 64 bytes. + * + * - Buffers: these buffers are used for various purposes that don't require + * any alignment or initialization before they're used. This means they can + * be moved around at no cost for a new compression. + * + * Allocating Memory: + * + * The various types of objects must be allocated in order, so they can be + * correctly packed into the workspace buffer. That order is: + * + * 1. Objects + * 2. Init once / Tables + * 3. Aligned / Tables + * 4. Buffers / Tables + * + * Attempts to reserve objects of different types out of order will fail. + */ +typedef struct { + void* workspace; + void* workspaceEnd; + + void* objectEnd; + void* tableEnd; + void* tableValidEnd; + void* allocStart; + void* initOnceStart; + + BYTE allocFailed; + int workspaceOversizedDuration; + ZSTD_cwksp_alloc_phase_e phase; + ZSTD_cwksp_static_alloc_e isStatic; +} ZSTD_cwksp; + +/*-************************************* +* Functions +***************************************/ + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws); + +MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { + (void)ws; + assert(ws->workspace <= ws->objectEnd); + assert(ws->objectEnd <= ws->tableEnd); + assert(ws->objectEnd <= ws->tableValidEnd); + assert(ws->tableEnd <= ws->allocStart); + assert(ws->tableValidEnd <= ws->allocStart); + assert(ws->allocStart <= ws->workspaceEnd); + assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws)); + assert(ws->workspace <= ws->initOnceStart); +#if ZSTD_MEMORY_SANITIZER + { + intptr_t const offset = __msan_test_shadow(ws->initOnceStart, + (U8*)ZSTD_cwksp_initialAllocStart(ws) - (U8*)ws->initOnceStart); +#if defined(ZSTD_MSAN_PRINT) + if(offset!=-1) { + __msan_print_shadow((U8*)ws->initOnceStart + offset - 8, 32); + } +#endif + assert(offset==-1); + }; +#endif +} + +/** + * Align must be a power of 2. + */ +MEM_STATIC size_t ZSTD_cwksp_align(size_t size, size_t const align) { + size_t const mask = align - 1; + assert((align & mask) == 0); + return (size + mask) & ~mask; +} + +/** + * Use this to determine how much space in the workspace we will consume to + * allocate this object. (Normally it should be exactly the size of the object, + * but under special conditions, like ASAN, where we pad each object, it might + * be larger.) + * + * Since tables aren't currently redzoned, you don't need to call through this + * to figure out how much space you need for the matchState tables. Everything + * else is though. + * + * Do not use for sizing aligned buffers. Instead, use ZSTD_cwksp_aligned_alloc_size(). + */ +MEM_STATIC size_t ZSTD_cwksp_alloc_size(size_t size) { + if (size == 0) + return 0; +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + return size + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#else + return size; +#endif +} + +/** + * Returns an adjusted alloc size that is the nearest larger multiple of 64 bytes. + * Used to determine the number of bytes required for a given "aligned". + */ +MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) { + return ZSTD_cwksp_alloc_size(ZSTD_cwksp_align(size, ZSTD_CWKSP_ALIGNMENT_BYTES)); +} + +/** + * Returns the amount of additional space the cwksp must allocate + * for internal purposes (currently only alignment). + */ +MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) { + /* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES + * bytes to align the beginning of tables section and end of buffers; + */ + size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; + return slackSpace; +} + + +/** + * Return the number of additional bytes required to align a pointer to the given number of bytes. + * alignBytes must be a power of two. + */ +MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignBytes) { + size_t const alignBytesMask = alignBytes - 1; + size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask; + assert((alignBytes & alignBytesMask) == 0); + assert(bytes < alignBytes); + return bytes; +} + +/** + * Returns the initial value for allocStart which is used to determine the position from + * which we can allocate from the end of the workspace. + */ +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) { + return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1)); +} + +/** + * Internal function. Do not use directly. + * Reserves the given number of bytes within the aligned/buffer segment of the wksp, + * which counts from the end of the wksp (as opposed to the object/table segment). + * + * Returns a pointer to the beginning of that space. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal_buffer_space(ZSTD_cwksp* ws, size_t const bytes) +{ + void* const alloc = (BYTE*)ws->allocStart - bytes; + void* const bottom = ws->tableEnd; + DEBUGLOG(5, "cwksp: reserving %p %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(alloc >= bottom); + if (alloc < bottom) { + DEBUGLOG(4, "cwksp: alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + /* the area is reserved from the end of wksp. + * If it overlaps with tableValidEnd, it voids guarantees on values' range */ + if (alloc < ws->tableValidEnd) { + ws->tableValidEnd = alloc; + } + ws->allocStart = alloc; + return alloc; +} + +/** + * Moves the cwksp to the next phase, and does any necessary allocations. + * cwksp initialization must necessarily go through each phase in order. + * Returns a 0 on success, or zstd error + */ +MEM_STATIC size_t +ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) +{ + assert(phase >= ws->phase); + if (phase > ws->phase) { + /* Going from allocating objects to allocating initOnce / tables */ + if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once && + phase >= ZSTD_cwksp_alloc_aligned_init_once) { + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + + { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */ + void *const alloc = ws->objectEnd; + size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES); + void *const objectEnd = (BYTE *) alloc + bytesToAlign; + DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign); + RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation, + "table phase - alignment initial allocation failed!"); + ws->objectEnd = objectEnd; + ws->tableEnd = objectEnd; /* table area starts being empty */ + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + } + } + ws->phase = phase; + ZSTD_cwksp_assert_internal_consistency(ws); + } + return 0; +} + +/** + * Returns whether this object/buffer/etc was allocated in this workspace. + */ +MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) +{ + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd); +} + +/** + * Internal function. Do not use directly. + */ +MEM_STATIC void* +ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) +{ + void* alloc; + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase)) || bytes == 0) { + return NULL; + } + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + bytes += 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + alloc = ZSTD_cwksp_reserve_internal_buffer_space(ws, bytes); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + if (alloc) { + alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + /* We need to keep the redzone poisoned while unpoisoning the bytes that + * are actually allocated. */ + __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE); + } + } +#endif + + return alloc; +} + +/** + * Reserves and returns unaligned memory. + */ +MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) +{ + return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); +} + +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + * This memory has been initialized at least once in the past. + * This doesn't mean it has been initialized this time, and it might contain data from previous + * operations. + * The main usage is for algorithms that might need read access into uninitialized memory. + * The algorithm must maintain safety under these conditions and must make sure it doesn't + * leak any of the past data (directly or in side channels). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES); + void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + if(ptr && ptr < ws->initOnceStart) { + /* We assume the memory following the current allocation is either: + * 1. Not usable as initOnce memory (end of workspace) + * 2. Another initOnce buffer that has been allocated before (and so was previously memset) + * 3. An ASAN redzone, in which case we don't want to write on it + * For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart. + * Note that we assume here that MSAN and ASAN cannot run in the same time. */ + ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes)); + ws->initOnceStart = ptr; + } +#if ZSTD_MEMORY_SANITIZER + assert(__msan_test_shadow(ptr, bytes) == -1); +#endif + return ptr; +} + +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) +{ + void* ptr = ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES), + ZSTD_cwksp_alloc_aligned); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + return ptr; +} + +/** + * Aligned on 64 bytes. These buffers have the special property that + * their values remain constrained, allowing us to re-use them without + * memset()-ing them. + */ +MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) +{ + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once; + void* alloc; + void* end; + void* top; + + /* We can only start allocating tables after we are done reserving space for objects at the + * start of the workspace */ + if(ws->phase < phase) { + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) { + return NULL; + } + } + alloc = ws->tableEnd; + end = (BYTE *)alloc + bytes; + top = ws->allocStart; + + DEBUGLOG(5, "cwksp: reserving %p table %zd bytes, %zd bytes remaining", + alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes); + assert((bytes & (sizeof(U32)-1)) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + assert(end <= top); + if (end > top) { + DEBUGLOG(4, "cwksp: table alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->tableEnd = end; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + assert((bytes & (ZSTD_CWKSP_ALIGNMENT_BYTES-1)) == 0); + assert(((size_t)alloc & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + return alloc; +} + +/** + * Aligned on sizeof(void*). + * Note : should happen only once, at workspace first initialization + */ +MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const roundedBytes = ZSTD_cwksp_align(bytes, sizeof(void*)); + void* alloc = ws->objectEnd; + void* end = (BYTE*)alloc + roundedBytes; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* over-reserve space */ + end = (BYTE *)end + 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE; +#endif + + DEBUGLOG(4, + "cwksp: reserving %p object %zd bytes (rounded to %zd), %zd bytes remaining", + alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes); + assert((size_t)alloc % ZSTD_ALIGNOF(void*) == 0); + assert(bytes % ZSTD_ALIGNOF(void*) == 0); + ZSTD_cwksp_assert_internal_consistency(ws); + /* we must be in the first phase, no advance is possible */ + if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) { + DEBUGLOG(3, "cwksp: object alloc failed!"); + ws->allocFailed = 1; + return NULL; + } + ws->objectEnd = end; + ws->tableEnd = end; + ws->tableValidEnd = end; + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* Move alloc so there's ZSTD_CWKSP_ASAN_REDZONE_SIZE unused space on + * either size. */ + alloc = (BYTE*)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + __asan_unpoison_memory_region(alloc, bytes); + } +#endif + + return alloc; +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) +{ + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_dirty"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the table re-use logic is sound, and that we don't + * access table space that we haven't cleaned, we re-"poison" the table + * space every time we mark it dirty. + * Since tableValidEnd space and initOnce space may overlap we don't poison + * the initOnce portion as it break its promise. This means that this poisoning + * check isn't always applied fully. */ + { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + assert(__msan_test_shadow(ws->objectEnd, size) == -1); + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + __msan_poison(ws->objectEnd, size); + } else { + assert(ws->initOnceStart >= ws->objectEnd); + __msan_poison(ws->objectEnd, (BYTE*)ws->initOnceStart - (BYTE*)ws->objectEnd); + } + } +#endif + + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + ws->tableValidEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_mark_tables_clean"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ws->tableValidEnd = ws->tableEnd; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Zero the part of the allocated tables not already marked clean. + */ +MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: ZSTD_cwksp_clean_tables"); + assert(ws->tableValidEnd >= ws->objectEnd); + assert(ws->tableValidEnd <= ws->allocStart); + if (ws->tableValidEnd < ws->tableEnd) { + ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd)); + } + ZSTD_cwksp_mark_tables_clean(ws); +} + +/** + * Invalidates table allocations. + * All other allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing tables!"); + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * Invalidates all buffer, aligned, and table allocations. + * Object allocations remain valid. + */ +MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { + DEBUGLOG(4, "cwksp: clearing!"); + +#if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) + /* To validate that the context re-use logic is sound, and that we don't + * access stuff that this compression hasn't initialized, we re-"poison" + * the workspace except for the areas in which we expect memory re-use + * without initialization (objects, valid tables area and init once + * memory). */ + { + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + size_t size = (BYTE*)ws->initOnceStart - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } + } +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined (ZSTD_ASAN_DONT_POISON_WORKSPACE) + /* We don't do this when the workspace is statically allocated, because + * when that is the case, we have no capability to hook into the end of the + * workspace's lifecycle to unpoison the memory. + */ + if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { + size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd; + __asan_poison_memory_region(ws->objectEnd, size); + } +#endif + + ws->tableEnd = ws->objectEnd; + ws->allocStart = ZSTD_cwksp_initialAllocStart(ws); + ws->allocFailed = 0; + if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) { + ws->phase = ZSTD_cwksp_alloc_aligned_init_once; + } + ZSTD_cwksp_assert_internal_consistency(ws); +} + +/** + * The provided workspace takes ownership of the buffer [start, start+size). + * Any existing values in the workspace are ignored (the previously managed + * buffer, if present, must be separately freed). + */ +MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_cwksp_static_alloc_e isStatic) { + DEBUGLOG(4, "cwksp: init'ing workspace with %zd bytes", size); + assert(((size_t)start & (sizeof(void*)-1)) == 0); /* ensure correct alignment */ + ws->workspace = start; + ws->workspaceEnd = (BYTE*)start + size; + ws->objectEnd = ws->workspace; + ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); + ws->phase = ZSTD_cwksp_alloc_objects; + ws->isStatic = isStatic; + ZSTD_cwksp_clear(ws); + ws->workspaceOversizedDuration = 0; + ZSTD_cwksp_assert_internal_consistency(ws); +} + +MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) { + void* workspace = ZSTD_customMalloc(size, customMem); + DEBUGLOG(4, "cwksp: creating new workspace with %zd bytes", size); + RETURN_ERROR_IF(workspace == NULL, memory_allocation, "NULL pointer!"); + ZSTD_cwksp_init(ws, workspace, size, ZSTD_cwksp_dynamic_alloc); + return 0; +} + +MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) { + void *ptr = ws->workspace; + DEBUGLOG(4, "cwksp: freeing workspace"); + ZSTD_memset(ws, 0, sizeof(ZSTD_cwksp)); + ZSTD_customFree(ptr, customMem); +} + +/** + * Moves the management of a workspace from one cwksp to another. The src cwksp + * is left in an invalid state (src must be re-init()'ed before it's used again). + */ +MEM_STATIC void ZSTD_cwksp_move(ZSTD_cwksp* dst, ZSTD_cwksp* src) { + *dst = *src; + ZSTD_memset(src, 0, sizeof(ZSTD_cwksp)); +} + +MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace); +} + +MEM_STATIC size_t ZSTD_cwksp_used(const ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->workspace) + + (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->allocStart); +} + +MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { + return ws->allocFailed; +} + +/*-************************************* +* Functions Checking Free Space +***************************************/ + +/* ZSTD_alignmentSpaceWithinBounds() : + * Returns if the estimated space needed for a wksp is within an acceptable limit of the + * actual amount of space used. + */ +MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) { + /* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice + * the alignment bytes difference between estimation and actual usage */ + return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) && + ZSTD_cwksp_used(ws) <= estimatedSpace; +} + + +MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) { + return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd); +} + +MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace; +} + +MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_available( + ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR); +} + +MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) { + return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace) + && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +MEM_STATIC void ZSTD_cwksp_bump_oversized_duration( + ZSTD_cwksp* ws, size_t additionalNeededSpace) { + if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) { + ws->workspaceOversizedDuration++; + } else { + ws->workspaceOversizedDuration = 0; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CWKSP_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.c new file mode 100644 index 000000000..0ad88ffc7 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.c @@ -0,0 +1,758 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_double_fast.h" + +static void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) { + ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i); + } + if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { + ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i); + } + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + +static void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHash = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHash = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) + hashSmall[smHash] = curr + i; + if (i == 0 || hashLarge[lgHash] == 0) + hashLarge[lgHash] = curr + i; + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm); + } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_noDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t mLength; + U32 offset; + U32 curr; + + /* how many positions to search before increasing step size */ + const size_t kStepIncr = 1 << kSearchStrength; + /* the position at which to increment the step size if no match is found */ + const BYTE* nextStep; + size_t step; /* the current step size */ + + size_t hl0; /* the long hash at ip */ + size_t hl1; /* the long hash at ip1 */ + + U32 idxl0; /* the long match index for ip */ + U32 idxl1; /* the long match index for ip1 */ + + const BYTE* matchl0; /* the long match for ip */ + const BYTE* matchs0; /* the short match for ip */ + const BYTE* matchl1; /* the long match for ip1 */ + + const BYTE* ip = istart; /* the current position */ + const BYTE* ip1; /* the next position */ + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_noDict_generic"); + + /* init */ + ip += ((ip - prefixLowest) == 0); + { + U32 const current = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); + U32 const maxRep = current - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* Outer Loop: one iteration per match found and stored */ + while (1) { + step = 1; + nextStep = ip + kStepIncr; + ip1 = ip + step; + + if (ip1 > ilimit) { + goto _cleanup; + } + + hl0 = ZSTD_hashPtr(ip, hBitsL, 8); + idxl0 = hashLong[hl0]; + matchl0 = base + idxl0; + + /* Inner Loop: one iteration per search / position */ + do { + const size_t hs0 = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 idxs0 = hashSmall[hs0]; + curr = (U32)(ip-base); + matchs0 = base + idxs0; + + hashLong[hl0] = hashSmall[hs0] = curr; /* update hash tables */ + + /* check noDict repcode */ + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + hl1 = ZSTD_hashPtr(ip1, hBitsL, 8); + + if (idxl0 > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchl0) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchl0+8, iend) + 8; + offset = (U32)(ip-matchl0); + while (((ip>anchor) & (matchl0>prefixLowest)) && (ip[-1] == matchl0[-1])) { ip--; matchl0--; mLength++; } /* catch up */ + goto _match_found; + } + } + + idxl1 = hashLong[hl1]; + matchl1 = base + idxl1; + + if (idxs0 > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(matchs0) == MEM_read32(ip)) { + goto _search_next_long; + } + } + + if (ip1 >= nextStep) { + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + step++; + nextStep += kStepIncr; + } + ip = ip1; + ip1 += step; + + hl0 = hl1; + idxl0 = idxl1; + matchl0 = matchl1; + #if defined(__aarch64__) + PREFETCH_L1(ip+256); + #endif + } while (ip1 <= ilimit); + +_cleanup: + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_search_next_long: + + /* check prefix long +1 match */ + if (idxl1 > prefixLowestIndex) { + if (MEM_read64(matchl1) == MEM_read64(ip1)) { + ip = ip1; + mLength = ZSTD_count(ip+8, matchl1+8, iend) + 8; + offset = (U32)(ip-matchl1); + while (((ip>anchor) & (matchl1>prefixLowest)) && (ip[-1] == matchl1[-1])) { ip--; matchl1--; mLength++; } /* catch up */ + goto _match_found; + } + } + + /* if no long +1 match, explore the short match we found */ + mLength = ZSTD_count(ip+4, matchs0+4, iend) + 4; + offset = (U32)(ip - matchs0); + while (((ip>anchor) & (matchs0>prefixLowest)) && (ip[-1] == matchs0[-1])) { ip--; matchs0--; mLength++; } /* catch up */ + + /* fall-through */ + +_match_found: /* requires ip, offset, mLength */ + offset_2 = offset_1; + offset_1 = offset; + + if (step < 4) { + /* It is unsafe to write this value back to the hashtable when ip1 is + * greater than or equal to the new ip we will have after we're done + * processing this match. Rather than perform that test directly + * (ip1 >= ip + mLength), which costs speed in practice, we do a simpler + * more predictable test. The minmatch even if we take a short match is + * 4 bytes, so as long as step, the distance between ip and ip1 + * (initially) is less than 4, we know ip1 < new ip. */ + hashLong[hl1] = (U32)(ip1 - base); + } + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + } + } +} + + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + const U32 hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + const U32 hBitsS = cParams->chainLog; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + /* presumes that, if there is a dictionary, it must be using Attach mode */ + const U32 prefixLowestIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixLowest = base + prefixLowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams; + const U32* const dictHashLong = dms->hashTable; + const U32* const dictHashSmall = dms->chainTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase); + const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); + + /* if a dictionary is attached, it must be within window range */ + assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32); + PREFETCH_AREA(dictHashLong, hashTableBytes) + PREFETCH_AREA(dictHashSmall, chainTableBytes) + } + + /* init */ + ip += (dictAndPrefixLength == 0); + + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + U32 offset; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS]; + U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL); + int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS); + U32 const curr = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + hashLong[h2] = hashSmall[h] = curr; /* update hash tables */ + + /* check repcode */ + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + goto _match_stored; + } + + if (matchIndexL > prefixLowestIndex) { + /* check prefix long match */ + if (MEM_read64(matchLong) == MEM_read64(ip)) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + goto _match_found; + } + } else if (dictTagsMatchL) { + /* check dictMatchState long match */ + U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatchL = dictBase + dictMatchIndexL; + assert(dictMatchL < dictEnd); + + if (dictMatchL > dictStart && MEM_read64(dictMatchL) == MEM_read64(ip)) { + mLength = ZSTD_count_2segments(ip+8, dictMatchL+8, iend, dictEnd, prefixLowest) + 8; + offset = (U32)(curr - dictMatchIndexL - dictIndexDelta); + while (((ip>anchor) & (dictMatchL>dictStart)) && (ip[-1] == dictMatchL[-1])) { ip--; dictMatchL--; mLength++; } /* catch up */ + goto _match_found; + } } + + if (matchIndexS > prefixLowestIndex) { + /* check prefix short match */ + if (MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } + } else if (dictTagsMatchS) { + /* check dictMatchState short match */ + U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS; + match = dictBase + dictMatchIndexS; + matchIndexS = dictMatchIndexS + dictIndexDelta; + + if (match > dictStart && MEM_read32(match) == MEM_read32(ip)) { + goto _search_next_long; + } } + + ip += ((ip-anchor) >> kSearchStrength) + 1; +#if defined(__aarch64__) + PREFETCH_L1(ip+256); +#endif + continue; + +_search_next_long: + { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8); + U32 const matchIndexL3 = hashLong[hl3]; + U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3); + const BYTE* matchL3 = base + matchIndexL3; + hashLong[hl3] = curr + 1; + + /* check prefix long +1 match */ + if (matchIndexL3 > prefixLowestIndex) { + if (MEM_read64(matchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count(ip+9, matchL3+8, iend) + 8; + ip++; + offset = (U32)(ip-matchL3); + while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ + goto _match_found; + } + } else if (dictTagsMatchL3) { + /* check dict long +1 match */ + U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; + assert(dictMatchL3 < dictEnd); + if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { + mLength = ZSTD_count_2segments(ip+1+8, dictMatchL3+8, iend, dictEnd, prefixLowest) + 8; + ip++; + offset = (U32)(curr + 1 - dictMatchIndexL3 - dictIndexDelta); + while (((ip>anchor) & (dictMatchL3>dictStart)) && (ip[-1] == dictMatchL3[-1])) { ip--; dictMatchL3--; mLength++; } /* catch up */ + goto _match_found; + } } } + + /* if no long +1 match, explore the short match we found */ + if (matchIndexS < prefixLowestIndex) { + mLength = ZSTD_count_2segments(ip+4, match+4, iend, dictEnd, prefixLowest) + 4; + offset = (U32)(curr - matchIndexS); + while (((ip>anchor) & (match>dictStart)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip - match); + while (((ip>anchor) & (match>prefixLowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + +_match_found: + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + +_match_stored: + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixLowestIndex ? + dictBase + repIndex2 - dictIndexDelta : + base + repIndex2; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } + } + } /* while (ip < ilimit) */ + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +#define ZSTD_GEN_DFAST_FN(dictMode, mls) \ + static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_doubleFast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls); \ + } + +ZSTD_GEN_DFAST_FN(noDict, 4) +ZSTD_GEN_DFAST_FN(noDict, 5) +ZSTD_GEN_DFAST_FN(noDict, 6) +ZSTD_GEN_DFAST_FN(noDict, 7) + +ZSTD_GEN_DFAST_FN(dictMatchState, 4) +ZSTD_GEN_DFAST_FN(dictMatchState, 5) +ZSTD_GEN_DFAST_FN(dictMatchState, 6) +ZSTD_GEN_DFAST_FN(dictMatchState, 7) + + +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize); + } +} + + +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + const U32 mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize); + } +} + + +static size_t ZSTD_compressBlock_doubleFast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls /* template */) +{ + ZSTD_compressionParameters const* cParams = &ms->cParams; + U32* const hashLong = ms->hashTable; + U32 const hBitsL = cParams->hashLog; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = (dictLimit > lowLimit) ? dictLimit : lowLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + U32 offset_1=rep[0], offset_2=rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_extDict_generic (srcSize=%zu)", srcSize); + + /* if extDict is invalidated due to maxDistance, switch to "regular" variant */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize); + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* const matchLongBase = matchLongIndex < prefixStartIndex ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 curr = (U32)(ip-base); + const U32 repIndex = curr + 1 - offset_1; /* offset_1 expected <= curr +1 */ + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = curr; /* update hash table */ + + if ((((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex doesn't overlap dict + prefix */ + & (offset_1 <= curr+1 - dictStartIndex)) /* note: we are searching at curr+1 */ + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; + ip++; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + } else { + if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchLongIndex < prefixStartIndex ? dictStart : prefixStart; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, prefixStart) + 8; + offset = curr - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + + } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < prefixStartIndex ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = curr + 1; + if ( (matchIndex3 > dictStartIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* const matchEnd = matchIndex3 < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex3 < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, prefixStart) + 8; + ip++; + offset = curr+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; + const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; + offset = curr - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + + } else { + ip += ((ip-anchor) >> kSearchStrength) + 1; + continue; + } } + + /* move to next sequence start */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Complementary insertion */ + /* done after iLimit test, as candidates could be > iend-8 */ + { U32 const indexToInsert = curr+2; + hashLong[ZSTD_hashPtr(base+indexToInsert, hBitsL, 8)] = indexToInsert; + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + hashSmall[ZSTD_hashPtr(base+indexToInsert, hBitsS, mls)] = indexToInsert; + hashSmall[ZSTD_hashPtr(ip-1, hBitsS, mls)] = (U32)(ip-1-base); + } + + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) /* intentional overflow : ensure repIndex2 doesn't overlap dict + prefix */ + & (offset_2 <= current2 - dictStartIndex)) + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +ZSTD_GEN_DFAST_FN(extDict, 4) +ZSTD_GEN_DFAST_FN(extDict, 5) +ZSTD_GEN_DFAST_FN(extDict, 6) +ZSTD_GEN_DFAST_FN(extDict, 7) + +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize); + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.h new file mode 100644 index 000000000..6f0047c4b --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_double_fast.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_DOUBLE_FAST_H +#define ZSTD_DOUBLE_FAST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); +size_t ZSTD_compressBlock_doubleFast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_doubleFast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_DOUBLE_FAST_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.c new file mode 100644 index 000000000..5f2c6a2ed --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.c @@ -0,0 +1,960 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ +#include "zstd_fast.h" + +static void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Currently, we always use ZSTD_dtlm_full for filling CDict tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_full); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls); + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); } + + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */ + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p); + } } } } +} + +static void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; + + /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_fast); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); + hashTable[hash0] = curr; + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hash] == 0) { /* not yet filled */ + hashTable[hash] = curr + p; + } } } } +} + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillHashTableForCCtx(ms, end, dtlm); + } +} + + +/** + * If you squint hard enough (and ignore repcodes), the search operation at any + * given position is broken into 4 stages: + * + * 1. Hash (map position to hash value via input read) + * 2. Lookup (map hash val to index via hashtable read) + * 3. Load (map index to value at that position via input read) + * 4. Compare + * + * Each of these steps involves a memory read at an address which is computed + * from the previous step. This means these steps must be sequenced and their + * latencies are cumulative. + * + * Rather than do 1->2->3->4 sequentially for a single position before moving + * onto the next, this implementation interleaves these operations across the + * next few positions: + * + * R = Repcode Read & Compare + * H = Hash + * T = Table Lookup + * M = Match Read & Compare + * + * Pos | Time --> + * ----+------------------- + * N | ... M + * N+1 | ... TM + * N+2 | R H T M + * N+3 | H TM + * N+4 | R H T M + * N+5 | H ... + * N+6 | R ... + * + * This is very much analogous to the pipelining of execution in a CPU. And just + * like a CPU, we have to dump the pipeline when we find a match (i.e., take a + * branch). + * + * When this happens, we throw away our current state, and do the following prep + * to re-enter the loop: + * + * Pos | Time --> + * ----+------------------- + * N | H T + * N+1 | H + * + * This is also the work we do at the beginning to enter the loop initially. + */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_fast_noDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, + U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = hasStep ? (cParams->targetLength + !(cParams->targetLength) + 1) : 2; + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 prefixStartIndex = ZSTD_getLowestPrefixIndex(ms, endIndex, cParams->windowLog); + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + + const BYTE* anchor = istart; + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + U32 rep_offset1 = rep[0]; + U32 rep_offset2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + U32 mval; /* src value at match idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + + /* ip0 and ip1 are always adjacent. The targetLength skipping and + * uncompressibility acceleration is applied to every other position, + * matching the behavior of #1562. step therefore represents the gap + * between pairs of positions, from ip0 to ip2 or ip1 to ip3. */ + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + + DEBUGLOG(5, "ZSTD_compressBlock_fast_generic"); + ip0 += (ip0 == prefixStart); + { U32 const curr = (U32)(ip0 - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); + U32 const maxRep = curr - windowLow; + if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0; + if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0; + } + + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + + do { + /* load repcode match for ip[2]*/ + const U32 rval = MEM_read32(ip2 - rep_offset1); + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if ((MEM_read32(ip2) == rval) & (rep_offset1 > 0)) { + ip0 = ip2; + match0 = ip0 - rep_offset1; + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + + /* First write next hash table entry; we've already calculated it. + * This write is known to be safe because the ip1 is before the + * repcode (ip2). */ + hashTable[hash1] = (U32)(ip1 - base); + + goto _match; + } + + /* load match for ip[0] */ + if (idx >= prefixStartIndex) { + mval = MEM_read32(base + idx); + } else { + mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */ + } + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + + /* First write next hash table entry; we've already calculated it. + * This write is known to be safe because the ip1 == ip0 + 1, so + * we know we will resume searching after ip1 */ + hashTable[hash1] = (U32)(ip1 - base); + + goto _offset; + } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* load match for ip[0] */ + if (idx >= prefixStartIndex) { + mval = MEM_read32(base + idx); + } else { + mval = MEM_read32(ip0) ^ 1; /* guaranteed to not match. */ + } + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + + /* first write next hash table entry; we've already calculated it */ + if (step <= 4) { + /* We need to avoid writing an index into the hash table >= the + * position at which we will pick up our searching after we've + * taken this match. + * + * The minimum possible match has length 4, so the earliest ip0 + * can be after we take this match will be the current ip0 + 4. + * ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely + * write this position. + */ + hashTable[hash1] = (U32)(ip1 - base); + } + + goto _offset; + } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* When the repcodes are outside of the prefix, we set them to zero before the loop. + * When the offsets are still zero, we need to restore them after the block to have a correct + * repcode history. If only one offset was invalid, it is easy. The tricky case is when both + * offsets were invalid. We need to figure out which offset to refill with. + * - If both offsets are zero they are in the same order. + * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`. + * - If only one is zero, we need to decide which offset to restore. + * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1. + * - It is impossible for rep_offset2 to be non-zero. + * + * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then + * set rep[0] = rep_offset1 and rep[1] = offsetSaved1. + */ + offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1; + rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx */ + + /* Compute the offset code. */ + match0 = base + idx; + rep_offset2 = rep_offset1; + rep_offset1 = (U32)(ip0-match0); + offcode = OFFSET_TO_OFFBASE(rep_offset1); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>prefixStart)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } + +_match: /* Requires: ip0, match0, offcode */ + + /* Count the forward length. */ + mLength += ZSTD_count(ip0 + mLength, match0 + mLength, iend); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + if (rep_offset2 > 0) { /* rep_offset2==0 means rep_offset2 is invalidated */ + while ( (ip0 <= ilimit) && (MEM_read32(ip0) == MEM_read32(ip0 - rep_offset2)) ) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip0+4, ip0+4-rep_offset2, iend) + 4; + { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += rLength; + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength); + anchor = ip0; + continue; /* faster when present (confirmed on gcc-8) ... (?) */ + } } } + + goto _start; +} + +#define ZSTD_GEN_FAST_FN(dictMode, mls, step) \ + static size_t ZSTD_compressBlock_fast_##dictMode##_##mls##_##step( \ + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \ + void const* src, size_t srcSize) \ + { \ + return ZSTD_compressBlock_fast_##dictMode##_generic(ms, seqStore, rep, src, srcSize, mls, step); \ + } + +ZSTD_GEN_FAST_FN(noDict, 4, 1) +ZSTD_GEN_FAST_FN(noDict, 5, 1) +ZSTD_GEN_FAST_FN(noDict, 6, 1) +ZSTD_GEN_FAST_FN(noDict, 7, 1) + +ZSTD_GEN_FAST_FN(noDict, 4, 0) +ZSTD_GEN_FAST_FN(noDict, 5, 0) +ZSTD_GEN_FAST_FN(noDict, 6, 0) +ZSTD_GEN_FAST_FN(noDict, 7, 0) + +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + if (ms->cParams.targetLength > 1) { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_1(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_1(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_1(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_1(ms, seqStore, rep, src, srcSize); + } + } else { + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_noDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_noDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_noDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_noDict_7_0(ms, seqStore, rep, src, srcSize); + } + + } +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_fast_dictMatchState_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + const BYTE* const base = ms->window.base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip0 = istart; + const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */ + const BYTE* anchor = istart; + const U32 prefixStartIndex = ms->window.dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=rep[0], offset_2=rep[1]; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; + const U32* const dictHashTable = dms->hashTable; + const U32 dictStartIndex = dms->window.dictLimit; + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictStart = dictBase + dictStartIndex; + const BYTE* const dictEnd = dms->window.nextSrc; + const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); + const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart); + const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + + /* if a dictionary is still attached, it necessarily means that + * it is within window size. So we just check it. */ + const U32 maxDistance = 1U << cParams->windowLog; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + assert(endIndex - prefixStartIndex <= maxDistance); + (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ + + (void)hasStep; /* not currently specialized on whether it's accelerated */ + + /* ensure there will be no underflow + * when translating a dict index into a local index */ + assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); + + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + PREFETCH_AREA(dictHashTable, hashTableBytes) + } + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); + ip0 += (dictAndPrefixLength == 0); + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + + /* Outer search loop */ + assert(stepSize >= 1); + while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */ + size_t mLength; + size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls); + + size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls); + U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0); + + U32 matchIndex = hashTable[hash0]; + U32 curr = (U32)(ip0 - base); + size_t step = stepSize; + const size_t kStepIncr = 1 << kSearchStrength; + const BYTE* nextStep = ip0 + kStepIncr; + + /* Inner search loop */ + while (1) { + const BYTE* match = base + matchIndex; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixStartIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls); + size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls); + hashTable[hash0] = curr; /* update hash table */ + + if (((U32) ((prefixStartIndex - 1) - repIndex) >= + 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ + && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip0++; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + break; + } + + if (dictTagsMatch) { + /* Found a possible dict match */ + const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex > dictStartIndex && + MEM_read32(dictMatch) == MEM_read32(ip0)) { + /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */ + if (matchIndex <= prefixStartIndex) { + U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta); + mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4; + while (((ip0 > anchor) & (dictMatch > dictStart)) + && (ip0[-1] == dictMatch[-1])) { + ip0--; + dictMatch--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + } + } + + if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) { + /* found a regular match */ + U32 const offset = (U32) (ip0 - match); + mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4; + while (((ip0 > anchor) & (match > prefixStart)) + && (ip0[-1] == match[-1])) { + ip0--; + match--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + + /* Prepare for next iteration */ + dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS]; + dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1); + matchIndex = hashTable[hash1]; + + if (ip1 >= nextStep) { + step++; + nextStep += kStepIncr; + } + ip0 = ip1; + ip1 = ip1 + step; + if (ip1 > ilimit) goto _cleanup; + + curr = (U32)(ip0 - base); + hash0 = hash1; + } /* end inner search loop */ + + /* match found */ + assert(mLength); + ip0 += mLength; + anchor = ip0; + + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+curr+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + /* check immediate repcode */ + while (ip0 <= ilimit) { + U32 const current2 = (U32)(ip0-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? + dictBase - dictIndexDelta + repIndex2 : + base + repIndex2; + if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch2) == MEM_read32(ip0))) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2; + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } + } + + /* Prepare for next iteration */ + assert(ip0 == anchor); + ip1 = ip0 + stepSize; + } + +_cleanup: + /* save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +ZSTD_GEN_FAST_FN(dictMatchState, 4, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 5, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 6, 0) +ZSTD_GEN_FAST_FN(dictMatchState, 7, 0) + +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState != NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_dictMatchState_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_dictMatchState_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_dictMatchState_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_dictMatchState_7_0(ms, seqStore, rep, src, srcSize); + } +} + + +static size_t ZSTD_compressBlock_fast_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize, U32 const mls, U32 const hasStep) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hlog = cParams->hashLog; + /* support stepSize of 0 */ + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* anchor = istart; + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); + const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); + const U32 dictStartIndex = lowLimit; + const BYTE* const dictStart = dictBase + dictStartIndex; + const U32 dictLimit = ms->window.dictLimit; + const U32 prefixStartIndex = dictLimit < lowLimit ? lowLimit : dictLimit; + const BYTE* const prefixStart = base + prefixStartIndex; + const BYTE* const dictEnd = dictBase + prefixStartIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + const BYTE* idxBase; /* base pointer for idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */ + + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); + + (void)hasStep; /* not currently specialized on whether it's accelerated */ + + DEBUGLOG(5, "ZSTD_compressBlock_fast_extDict_generic (offset_1=%u)", offset_1); + + /* switch to "regular" variant if extDict is invalidated due to maxDistance */ + if (prefixStartIndex == dictStartIndex) + return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); + + { U32 const curr = (U32)(ip0 - base); + U32 const maxRep = curr - dictStartIndex; + if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + do { + { /* load repcode match for ip[2] */ + U32 const current2 = (U32)(ip2 - base); + U32 const repIndex = current2 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + U32 rval; + if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */ + & (offset_1 > 0) ) { + rval = MEM_read32(repBase + repIndex); + } else { + rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */ + } + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if (MEM_read32(ip2) == rval) { + ip0 = ip2; + match0 = repBase + repIndex; + matchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + assert((match0 != prefixStart) & (match0 != dictStart)); + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + goto _match; + } } + + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx, idxBase */ + + /* Compute the offset code. */ + { U32 const offset = current0 - idx; + const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart; + matchEnd = idx < prefixStartIndex ? dictEnd : iend; + match0 = idxBase + idx; + offset_2 = offset_1; + offset_1 = offset; + offcode = OFFSET_TO_OFFBASE(offset); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } } + +_match: /* Requires: ip0, match0, offcode, matchEnd */ + + /* Count the forward length. */ + assert(matchEnd != 0); + mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* write next hash table entry */ + if (ip1 < ip0) { + hashTable[hash1] = (U32)(ip1 - base); + } + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + while (ip0 <= ilimit) { + U32 const repIndex2 = (U32)(ip0-base) - offset_2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0)) /* intentional underflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } } + + goto _start; +} + +ZSTD_GEN_FAST_FN(extDict, 4, 0) +ZSTD_GEN_FAST_FN(extDict, 5, 0) +ZSTD_GEN_FAST_FN(extDict, 6, 0) +ZSTD_GEN_FAST_FN(extDict, 7, 0) + +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); + switch(mls) + { + default: /* includes case 3 */ + case 4 : + return ZSTD_compressBlock_fast_extDict_4_0(ms, seqStore, rep, src, srcSize); + case 5 : + return ZSTD_compressBlock_fast_extDict_5_0(ms, seqStore, rep, src, srcSize); + case 6 : + return ZSTD_compressBlock_fast_extDict_6_0(ms, seqStore, rep, src, srcSize); + case 7 : + return ZSTD_compressBlock_fast_extDict_7_0(ms, seqStore, rep, src, srcSize); + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.h new file mode 100644 index 000000000..9e4236b47 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_fast.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_FAST_H +#define ZSTD_FAST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "../common/mem.h" /* U32 */ +#include "zstd_compress_internal.h" + +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); +size_t ZSTD_compressBlock_fast( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_fast_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c new file mode 100644 index 000000000..5ba88e867 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.c @@ -0,0 +1,2157 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "zstd_lazy.h" +#include "../common/bits.h" /* ZSTD_countTrailingZeros64 */ + +#define kLazySkippingStep 8 + + +/*-************************************* +* Binary Tree search +***************************************/ + +static void +ZSTD_updateDUBT(ZSTD_matchState_t* ms, + const BYTE* ip, const BYTE* iend, + U32 mls) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + if (idx != target) + DEBUGLOG(7, "ZSTD_updateDUBT, from %u to %u (dictLimit:%u)", + idx, target, ms->window.dictLimit); + assert(ip + 8 <= iend); /* condition for ZSTD_hashPtr */ + (void)iend; + + assert(idx >= ms->window.dictLimit); /* condition for valid base+idx */ + for ( ; idx < target ; idx++) { + size_t const h = ZSTD_hashPtr(base + idx, hashLog, mls); /* assumption : ip + 8 <= iend */ + U32 const matchIndex = hashTable[h]; + + U32* const nextCandidatePtr = bt + 2*(idx&btMask); + U32* const sortMarkPtr = nextCandidatePtr + 1; + + DEBUGLOG(8, "ZSTD_updateDUBT: insert %u", idx); + hashTable[h] = idx; /* Update Hash Table */ + *nextCandidatePtr = matchIndex; /* update BT like a chain */ + *sortMarkPtr = ZSTD_DUBT_UNSORTED_MARK; + } + ms->nextToUpdate = target; +} + + +/** ZSTD_insertDUBT1() : + * sort one already inserted but unsorted position + * assumption : curr >= btlow == (curr - btmask) + * doesn't fail */ +static void +ZSTD_insertDUBT1(const ZSTD_matchState_t* ms, + U32 curr, const BYTE* inputEnd, + U32 nbCompares, U32 btLow, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const ip = (curr>=dictLimit) ? base + curr : dictBase + curr; + const BYTE* const iend = (curr>=dictLimit) ? inputEnd : dictBase + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ + U32 dummy32; /* to be nullified at the end */ + U32 const windowValid = ms->window.lowLimit; + U32 const maxDistance = 1U << cParams->windowLog; + U32 const windowLow = (curr - windowValid > maxDistance) ? curr - maxDistance : windowValid; + + + DEBUGLOG(8, "ZSTD_insertDUBT1(%u) (dictLimit=%u, lowLimit=%u)", + curr, dictLimit, windowLow); + assert(curr >= btLow); + assert(ip < iend); /* condition for ZSTD_count */ + + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + /* note : all candidates are now supposed sorted, + * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK + * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ + + if ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ + || (curr < dictLimit) /* both in extDict */) { + const BYTE* const mBase = ( (dictMode != ZSTD_extDict) + || (matchIndex+matchLength >= dictLimit)) ? + base : dictBase; + assert( (matchIndex+matchLength >= dictLimit) /* might be wrong if extDict is incorrectly set to 0 */ + || (curr < dictLimit) ); + match = mBase + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* preparation for next read of match[matchLength] */ + } + + DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", + curr, matchIndex, (U32)matchLength); + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is smaller : next => %u", + matchIndex, btLow, nextPtr[1]); + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + DEBUGLOG(8, "ZSTD_insertDUBT1: %u (>btLow=%u) is larger => %u", + matchIndex, btLow, nextPtr[0]); + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; +} + + +static size_t +ZSTD_DUBT_findBetterDictMatch ( + const ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + size_t bestLength, + U32 nbCompares, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_matchState_t * const dms = ms->dictMatchState; + const ZSTD_compressionParameters* const dmsCParams = &dms->cParams; + const U32 * const dictHashTable = dms->hashTable; + U32 const hashLog = dmsCParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 dictMatchIndex = dictHashTable[h]; + + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + U32 const curr = (U32)(ip-base); + const BYTE* const dictBase = dms->window.base; + const BYTE* const dictEnd = dms->window.nextSrc; + U32 const dictHighLimit = (U32)(dms->window.nextSrc - dms->window.base); + U32 const dictLowLimit = dms->window.lowLimit; + U32 const dictIndexDelta = ms->window.lowLimit - dictHighLimit; + + U32* const dictBt = dms->chainTable; + U32 const btLog = dmsCParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= dictHighLimit - dictLowLimit) ? dictLowLimit : dictHighLimit - btMask; + + size_t commonLengthSmaller=0, commonLengthLarger=0; + + (void)dictMode; + assert(dictMode == ZSTD_dictMatchState); + + for (; nbCompares && (dictMatchIndex > dictLowLimit); --nbCompares) { + U32* const nextPtr = dictBt + 2*(dictMatchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dictBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (dictMatchIndex+matchLength >= dictHighLimit) + match = base + dictMatchIndex + dictIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + U32 matchIndex = dictMatchIndex + dictIndexDelta; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { + DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + } + if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + if (dictMatchIndex <= btLow) { break; } /* beyond tree size, stop the search */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } + } + + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + } + return bestLength; + +} + + +static size_t +ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + size_t* offBasePtr, + U32 const mls, + const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const unsortLimit = MAX(btLow, windowLow); + + U32* nextCandidate = bt + 2*(matchIndex&btMask); + U32* unsortedMark = bt + 2*(matchIndex&btMask) + 1; + U32 nbCompares = 1U << cParams->searchLog; + U32 nbCandidates = nbCompares; + U32 previousCandidate = 0; + + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch (%u) ", curr); + assert(ip <= iend-8); /* required for h calculation */ + assert(dictMode != ZSTD_dedicatedDictSearch); + + /* reach end of unsorted candidates list */ + while ( (matchIndex > unsortLimit) + && (*unsortedMark == ZSTD_DUBT_UNSORTED_MARK) + && (nbCandidates > 1) ) { + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", + matchIndex); + *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ + previousCandidate = matchIndex; + matchIndex = *nextCandidate; + nextCandidate = bt + 2*(matchIndex&btMask); + unsortedMark = bt + 2*(matchIndex&btMask) + 1; + nbCandidates --; + } + + /* nullify last candidate if it's still unsorted + * simplification, detrimental to compression ratio, beneficial for speed */ + if ( (matchIndex > unsortLimit) + && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) { + DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", + matchIndex); + *nextCandidate = *unsortedMark = 0; + } + + /* batch sort stacked candidates */ + matchIndex = previousCandidate; + while (matchIndex) { /* will end on matchIndex == 0 */ + U32* const nextCandidateIdxPtr = bt + 2*(matchIndex&btMask) + 1; + U32 const nextCandidateIdx = *nextCandidateIdxPtr; + ZSTD_insertDUBT1(ms, matchIndex, iend, + nbCandidates, unsortLimit, dictMode); + matchIndex = nextCandidateIdx; + nbCandidates++; + } + + /* find longest match */ + { size_t commonLengthSmaller = 0, commonLengthLarger = 0; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr + 8 + 1; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + matchIndex = hashTable[h]; + hashTable[h] = curr; /* Update Hash Table */ + + for (; nbCompares && (matchIndex > windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) ) + bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + if (dictMode == ZSTD_dictMatchState) { + nbCompares = 0; /* in addition to avoiding checking any + * further in this loop, make sure we + * skip checking in the dictionary. */ + } + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + bestLength = ZSTD_DUBT_findBetterDictMatch( + ms, ip, iend, + offBasePtr, bestLength, nbCompares, + mls, dictMode); + } + + assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + if (bestLength >= MINMATCH) { + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex; + DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", + curr, (U32)bestLength, (U32)*offBasePtr, mIndex); + } + return bestLength; + } +} + + +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offBasePtr, + const U32 mls /* template */, + const ZSTD_dictMode_e dictMode) +{ + DEBUGLOG(7, "ZSTD_BtFindBestMatch"); + if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateDUBT(ms, ip, iLimit, mls); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode); +} + +/*********************************** +* Dedicated dict search +***********************************/ + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32* const hashTable = ms->hashTable; + U32* const chainTable = ms->chainTable; + U32 const chainSize = 1 << ms->cParams.chainLog; + U32 idx = ms->nextToUpdate; + U32 const minChain = chainSize < target - idx ? target - chainSize : idx; + U32 const bucketSize = 1 << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const cacheSize = bucketSize - 1; + U32 const chainAttempts = (1 << ms->cParams.searchLog) - cacheSize; + U32 const chainLimit = chainAttempts > 255 ? 255 : chainAttempts; + + /* We know the hashtable is oversized by a factor of `bucketSize`. + * We are going to temporarily pretend `bucketSize == 1`, keeping only a + * single entry. We will use the rest of the space to construct a temporary + * chaintable. + */ + U32 const hashLog = ms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + U32* const tmpHashTable = hashTable; + U32* const tmpChainTable = hashTable + ((size_t)1 << hashLog); + U32 const tmpChainSize = (U32)((1 << ZSTD_LAZY_DDSS_BUCKET_LOG) - 1) << hashLog; + U32 const tmpMinChain = tmpChainSize < target ? target - tmpChainSize : idx; + U32 hashIdx; + + assert(ms->cParams.chainLog <= 24); + assert(ms->cParams.hashLog > ms->cParams.chainLog); + assert(idx != 0); + assert(tmpMinChain <= minChain); + + /* fill conventional hash table and conventional chain table */ + for ( ; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch); + if (idx >= tmpMinChain) { + tmpChainTable[idx - tmpMinChain] = hashTable[h]; + } + tmpHashTable[h] = idx; + } + + /* sort chains into ddss chain table */ + { + U32 chainPos = 0; + for (hashIdx = 0; hashIdx < (1U << hashLog); hashIdx++) { + U32 count; + U32 countBeyondMinChain = 0; + U32 i = tmpHashTable[hashIdx]; + for (count = 0; i >= tmpMinChain && count < cacheSize; count++) { + /* skip through the chain to the first position that won't be + * in the hash cache bucket */ + if (i < minChain) { + countBeyondMinChain++; + } + i = tmpChainTable[i - tmpMinChain]; + } + if (count == cacheSize) { + for (count = 0; count < chainLimit;) { + if (i < minChain) { + if (!i || ++countBeyondMinChain > cacheSize) { + /* only allow pulling `cacheSize` number of entries + * into the cache or chainTable beyond `minChain`, + * to replace the entries pulled out of the + * chainTable into the cache. This lets us reach + * back further without increasing the total number + * of entries in the chainTable, guaranteeing the + * DDSS chain table will fit into the space + * allocated for the regular one. */ + break; + } + } + chainTable[chainPos++] = i; + count++; + if (i < tmpMinChain) { + break; + } + i = tmpChainTable[i - tmpMinChain]; + } + } else { + count = 0; + } + if (count) { + tmpHashTable[hashIdx] = ((chainPos - count) << 8) + count; + } else { + tmpHashTable[hashIdx] = 0; + } + } + assert(chainPos <= chainSize); /* I believe this is guaranteed... */ + } + + /* move chain pointers into the last entry of each hash bucket */ + for (hashIdx = (1 << hashLog); hashIdx; ) { + U32 const bucketIdx = --hashIdx << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 const chainPackedPointer = tmpHashTable[hashIdx]; + U32 i; + for (i = 0; i < cacheSize; i++) { + hashTable[bucketIdx + i] = 0; + } + hashTable[bucketIdx + bucketSize - 1] = chainPackedPointer; + } + + /* fill the buckets of the hash table */ + for (idx = ms->nextToUpdate; idx < target; idx++) { + U32 const h = (U32)ZSTD_hashPtr(base + idx, hashLog, ms->cParams.minMatch) + << ZSTD_LAZY_DDSS_BUCKET_LOG; + U32 i; + /* Shift hash cache down 1. */ + for (i = cacheSize - 1; i; i--) + hashTable[h + i] = hashTable[h + i - 1]; + hashTable[h] = idx; + } + + ms->nextToUpdate = target; +} + +/* Returns the longest match length found in the dedicated dict search structure. + * If none are longer than the argument ml, then ml will be returned. + */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nbAttempts, + const ZSTD_matchState_t* const dms, + const BYTE* const ip, const BYTE* const iLimit, + const BYTE* const prefixStart, const U32 curr, + const U32 dictLimit, const size_t ddsIdx) { + const U32 ddsLowestIndex = dms->window.dictLimit; + const BYTE* const ddsBase = dms->window.base; + const BYTE* const ddsEnd = dms->window.nextSrc; + const U32 ddsSize = (U32)(ddsEnd - ddsBase); + const U32 ddsIndexDelta = dictLimit - ddsSize; + const U32 bucketSize = (1 << ZSTD_LAZY_DDSS_BUCKET_LOG); + const U32 bucketLimit = nbAttempts < bucketSize - 1 ? nbAttempts : bucketSize - 1; + U32 ddsAttempt; + U32 matchIndex; + + for (ddsAttempt = 0; ddsAttempt < bucketSize - 1; ddsAttempt++) { + PREFETCH_L1(ddsBase + dms->hashTable[ddsIdx + ddsAttempt]); + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 const chainIndex = chainPackedPointer >> 8; + + PREFETCH_L1(&dms->chainTable[chainIndex]); + } + + for (ddsAttempt = 0; ddsAttempt < bucketLimit; ddsAttempt++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->hashTable[ddsIdx + ddsAttempt]; + match = ddsBase + matchIndex; + + if (!matchIndex) { + return ml; + } + + /* guaranteed by table construction */ + (void)ddsLowestIndex; + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) { + /* best possible, avoids read overflow on next attempt */ + return ml; + } + } + } + + { + U32 const chainPackedPointer = dms->hashTable[ddsIdx + bucketSize - 1]; + U32 chainIndex = chainPackedPointer >> 8; + U32 const chainLength = chainPackedPointer & 0xFF; + U32 const chainAttempts = nbAttempts - ddsAttempt; + U32 const chainLimit = chainAttempts > chainLength ? chainLength : chainAttempts; + U32 chainAttempt; + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++) { + PREFETCH_L1(ddsBase + dms->chainTable[chainIndex + chainAttempt]); + } + + for (chainAttempt = 0 ; chainAttempt < chainLimit; chainAttempt++, chainIndex++) { + size_t currentMl=0; + const BYTE* match; + matchIndex = dms->chainTable[chainIndex]; + match = ddsBase + matchIndex; + + /* guaranteed by table construction */ + assert(matchIndex >= ddsLowestIndex); + assert(match+4 <= ddsEnd); + if (MEM_read32(match) == MEM_read32(ip)) { + /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, ddsEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + return ml; +} + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( + ZSTD_matchState_t* ms, + const ZSTD_compressionParameters* const cParams, + const BYTE* ip, U32 const mls, U32 const lazySkipping) +{ + U32* const hashTable = ms->hashTable; + const U32 hashLog = cParams->hashLog; + U32* const chainTable = ms->chainTable; + const U32 chainMask = (1 << cParams->chainLog) - 1; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + /* Stop inserting every position when in the lazy skipping mode. */ + if (lazySkipping) + break; + } + + ms->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { + const ZSTD_compressionParameters* const cParams = &ms->cParams; + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0); +} + +/* inlining is important to hardwire a hot branch (template emulation) */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_HcFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const chainTable = ms->chainTable; + const U32 chainSize = (1 << cParams->chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 minChain = curr > chainSize ? curr - chainSize : 0; + U32 nbAttempts = 1U << cParams->searchLog; + size_t ml=4-1; + + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 ddsHashLog = dictMode == ZSTD_dedicatedDictSearch + ? dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + const size_t ddsIdx = dictMode == ZSTD_dedicatedDictSearch + ? ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG : 0; + + U32 matchIndex; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32* entry = &dms->hashTable[ddsIdx]; + PREFETCH_L1(entry); + } + + /* HC4 match finder */ + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping); + + for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + const U32* const dmsChainTable = dms->chainTable; + const U32 dmsChainSize = (1 << dms->cParams.chainLog); + const U32 dmsChainMask = dmsChainSize - 1; + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + const U32 dmsMinChain = dmsSize > dmsChainSize ? dmsSize - dmsChainSize : 0; + + matchIndex = dms->hashTable[ZSTD_hashPtr(ip, dms->cParams.hashLog, mls)]; + + for ( ; (matchIndex>=dmsLowestIndex) & (nbAttempts>0) ; nbAttempts--) { + size_t currentMl=0; + const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + + /* save best solution */ + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + + if (matchIndex <= dmsMinChain) break; + + matchIndex = dmsChainTable[matchIndex & dmsChainMask]; + } + } + + return ml; +} + +/* ********************************* +* (SIMD) Row-based matchfinder +***********************************/ +/* Constants for row-based hash */ +#define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1) +#define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */ + +#define ZSTD_ROW_HASH_CACHE_MASK (ZSTD_ROW_HASH_CACHE_SIZE - 1) + +typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 representing a mask of matches */ + +/* ZSTD_VecMask_next(): + * Starting from the LSB, returns the idx of the next non-zero bit. + * Basically counting the nb of trailing zeroes. + */ +MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) { + return ZSTD_countTrailingZeros64(val); +} + +/* ZSTD_row_nextIndex(): + * Returns the next index to insert at within a tagTable row, and updates the "head" + * value to reflect the update. Essentially cycles backwards from [1, {entries per row}) + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) { + U32 next = (*tagRow-1) & rowMask; + next += (next == 0) ? rowMask : 0; /* skip first position */ + *tagRow = (BYTE)next; + return next; +} + +/* ZSTD_isAligned(): + * Checks that a pointer is aligned to "align" bytes which must be a power of 2. + */ +MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) { + assert((align & (align - 1)) == 0); + return (((size_t)ptr) & (align - 1)) == 0; +} + +/* ZSTD_row_prefetch(): + * Performs prefetching for the hashTable and tagTable at a given row. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) { + PREFETCH_L1(hashTable + relRow); + if (rowLog >= 5) { + PREFETCH_L1(hashTable + relRow + 16); + /* Note: prefetching more of the hash table does not appear to be beneficial for 128-entry rows */ + } + PREFETCH_L1(tagTable + relRow); + if (rowLog == 6) { + PREFETCH_L1(tagTable + relRow + 32); + } + assert(rowLog == 4 || rowLog == 5 || rowLog == 6); + assert(ZSTD_isAligned(hashTable + relRow, 64)); /* prefetched hash row always 64-byte aligned */ + assert(ZSTD_isAligned(tagTable + relRow, (size_t)1 << rowLog)); /* prefetched tagRow sits on correct multiple of bytes (32,64,128) */ +} + +/* ZSTD_row_fillHashCache(): + * Fill up the hash cache starting at idx, prefetching up to ZSTD_ROW_HASH_CACHE_SIZE entries, + * but not beyond iLimit. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const BYTE* base, + U32 const rowLog, U32 const mls, + U32 idx, const BYTE* const iLimit) +{ + U32 const* const hashTable = ms->hashTable; + BYTE const* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1); + U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch); + + for (; idx < lim; ++idx) { + U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash; + } + + DEBUGLOG(6, "ZSTD_row_fillHashCache(): [%u %u %u %u %u %u %u %u]", ms->hashCache[0], ms->hashCache[1], + ms->hashCache[2], ms->hashCache[3], ms->hashCache[4], + ms->hashCache[5], ms->hashCache[6], ms->hashCache[7]); +} + +/* ZSTD_row_nextCachedHash(): + * Returns the hash of base + idx, and replaces the hash in the hash cache with the byte at + * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable. + */ +FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable, + BYTE const* tagTable, BYTE const* base, + U32 idx, U32 const hashLog, + U32 const rowLog, U32 const mls, + U64 const hashSalt) +{ + U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); + { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK]; + cache[idx & ZSTD_ROW_HASH_CACHE_MASK] = newHash; + return hash; + } +} + +/* ZSTD_row_update_internalImpl(): + * Updates the hash table with positions starting from updateStartIdx until updateEndIdx. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms, + U32 updateStartIdx, U32 const updateEndIdx, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) +{ + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32 const hashLog = ms->rowHashLog; + const BYTE* const base = ms->window.base; + + DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx); + for (; updateStartIdx < updateEndIdx; ++updateStartIdx) { + U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt) + : (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32* const row = hashTable + relRow; + BYTE* tagRow = tagTable + relRow; + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + + assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt)); + tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK; + row[pos] = updateStartIdx; + } +} + +/* ZSTD_row_update_internal(): + * Inserts the byte at ip into the appropriate position in the hash table, and updates ms->nextToUpdate. + * Skips sections of long matches as is necessary. + */ +FORCE_INLINE_TEMPLATE void ZSTD_row_update_internal(ZSTD_matchState_t* ms, const BYTE* ip, + U32 const mls, U32 const rowLog, + U32 const rowMask, U32 const useCache) +{ + U32 idx = ms->nextToUpdate; + const BYTE* const base = ms->window.base; + const U32 target = (U32)(ip - base); + const U32 kSkipThreshold = 384; + const U32 kMaxMatchStartPositionsToUpdate = 96; + const U32 kMaxMatchEndPositionsToUpdate = 32; + + if (useCache) { + /* Only skip positions when using hash cache, i.e. + * if we are loading a dict, don't skip anything. + * If we decide to skip, then we only update a set number + * of positions at the beginning and end of the match. + */ + if (UNLIKELY(target - idx > kSkipThreshold)) { + U32 const bound = idx + kMaxMatchStartPositionsToUpdate; + ZSTD_row_update_internalImpl(ms, idx, bound, mls, rowLog, rowMask, useCache); + idx = target - kMaxMatchEndPositionsToUpdate; + ZSTD_row_fillHashCache(ms, base, rowLog, mls, idx, ip+1); + } + } + assert(target >= idx); + ZSTD_row_update_internalImpl(ms, idx, target, mls, rowLog, rowMask, useCache); + ms->nextToUpdate = target; +} + +/* ZSTD_row_update(): + * External wrapper for ZSTD_row_update_internal(). Used for filling the hashtable during dictionary + * processing. + */ +void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) { + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + const U32 rowMask = (1u << rowLog) - 1; + const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */); + + DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog); + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */); +} + +/* Returns the mask width of bits group of which will be set to 1. Given not all + * architectures have easy movemask instruction, this helps to iterate over + * groups of bits easier and faster. + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_row_matchMaskGroupWidth(const U32 rowEntries) +{ + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES); + (void)rowEntries; +#if defined(ZSTD_ARCH_ARM_NEON) + /* NEON path only works for little endian */ + if (!MEM_isLittleEndian()) { + return 1; + } + if (rowEntries == 16) { + return 4; + } + if (rowEntries == 32) { + return 2; + } + if (rowEntries == 64) { + return 1; + } +#endif + return 1; +} + +#if defined(ZSTD_ARCH_X86_SSE2) +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U32 head) +{ + const __m128i comparisonMask = _mm_set1_epi8((char)tag); + int matches[4] = {0}; + int i; + assert(nbChunks == 1 || nbChunks == 2 || nbChunks == 4); + for (i=0; i> chunkSize; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= (chunk * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } else { /* big endian: reverse bits during extraction */ + const size_t msb = xFF ^ (xFF >> 1); + const size_t extractMagic = (msb / 0x1FF) | msb; + do { + size_t chunk = MEM_readST(&src[i]); + chunk ^= splatChar; + chunk = (((chunk | x80) - x01) | chunk) & x80; + matches <<= chunkSize; + matches |= ((chunk >> 7) * extractMagic) >> shiftAmount; + i -= chunkSize; + } while (i >= 0); + } + matches = ~matches; + if (rowEntries == 16) { + return ZSTD_rotateRight_U16((U16)matches, headGrouped); + } else if (rowEntries == 32) { + return ZSTD_rotateRight_U32((U32)matches, headGrouped); + } else { + return ZSTD_rotateRight_U64((U64)matches, headGrouped); + } + } +#endif +} + +/* The high-level approach of the SIMD row based match finder is as follows: + * - Figure out where to insert the new entry: + * - Generate a hash from a byte along with an additional 1-byte "short hash". The additional byte is our "tag" + * - The hashTable is effectively split into groups or "rows" of 16 or 32 entries of U32, and the hash determines + * which row to insert into. + * - Determine the correct position within the row to insert the entry into. Each row of 16 or 32 can + * be considered as a circular buffer with a "head" index that resides in the tagTable. + * - Also insert the "tag" into the equivalent row and position in the tagTable. + * - Note: The tagTable has 17 or 33 1-byte entries per row, due to 16 or 32 tags, and 1 "head" entry. + * The 17 or 33 entry rows are spaced out to occur every 32 or 64 bytes, respectively, + * for alignment/performance reasons, leaving some bytes unused. + * - Use SIMD to efficiently compare the tags in the tagTable to the 1-byte "short hash" and + * generate a bitfield that we can cycle through to check the collisions in the hash table. + * - Pick the longest match. + */ +FORCE_INLINE_TEMPLATE +size_t ZSTD_RowFindBestMatch( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 mls, const ZSTD_dictMode_e dictMode, + const U32 rowLog) +{ + U32* const hashTable = ms->hashTable; + BYTE* const tagTable = ms->tagTable; + U32* const hashCache = ms->hashCache; + const U32 hashLog = ms->rowHashLog; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 curr = (U32)(ip-base); + const U32 maxDistance = 1U << cParams->windowLog; + const U32 lowestValid = ms->window.lowLimit; + const U32 withinMaxDistance = (curr - lowestValid > maxDistance) ? curr - maxDistance : lowestValid; + const U32 isDictionary = (ms->loadedDictEnd != 0); + const U32 lowLimit = isDictionary ? lowestValid : withinMaxDistance; + const U32 rowEntries = (1U << rowLog); + const U32 rowMask = rowEntries - 1; + const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */ + const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries); + const U64 hashSalt = ms->hashSalt; + U32 nbAttempts = 1U << cappedSearchLog; + size_t ml=4-1; + U32 hash; + + /* DMS/DDS variables that may be referenced laster */ + const ZSTD_matchState_t* const dms = ms->dictMatchState; + + /* Initialize the following variables to satisfy static analyzer */ + size_t ddsIdx = 0; + U32 ddsExtraAttempts = 0; /* cctx hash tables are limited in searches, but allow extra searches into DDS */ + U32 dmsTag = 0; + U32* dmsRow = NULL; + BYTE* dmsTagRow = NULL; + + if (dictMode == ZSTD_dedicatedDictSearch) { + const U32 ddsHashLog = dms->cParams.hashLog - ZSTD_LAZY_DDSS_BUCKET_LOG; + { /* Prefetch DDS hashtable entry */ + ddsIdx = ZSTD_hashPtr(ip, ddsHashLog, mls) << ZSTD_LAZY_DDSS_BUCKET_LOG; + PREFETCH_L1(&dms->hashTable[ddsIdx]); + } + ddsExtraAttempts = cParams->searchLog > rowLog ? 1U << (cParams->searchLog - rowLog) : 0; + } + + if (dictMode == ZSTD_dictMatchState) { + /* Prefetch DMS rows */ + U32* const dmsHashTable = dms->hashTable; + BYTE* const dmsTagTable = dms->tagTable; + U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK; + dmsTagRow = (BYTE*)(dmsTagTable + dmsRelRow); + dmsRow = dmsHashTable + dmsRelRow; + ZSTD_row_prefetch(dmsHashTable, dmsTagTable, dmsRelRow, rowLog); + } + + /* Update the hashTable and tagTable up to (but not including) ip */ + if (!ms->lazySkipping) { + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */); + hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt); + } else { + /* Stop inserting every position when in the lazy skipping mode. + * The hash cache is also not kept up to date in this mode. + */ + hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + ms->nextToUpdate = curr; + } + ms->hashSaltEntropy += hash; /* collect salt entropy */ + + { /* Get the hash for ip, compute the appropriate row */ + U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; + U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK; + U32* const row = hashTable + relRow; + BYTE* tagRow = (BYTE*)(tagTable + relRow); + U32 const headGrouped = (*tagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries); + + /* Cycle through the matches and prefetch */ + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = row[matchPos]; + if(matchPos == 0) continue; + assert(numMatches < rowEntries); + if (matchIndex < lowLimit) + break; + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + PREFETCH_L1(base + matchIndex); + } else { + PREFETCH_L1(dictBase + matchIndex); + } + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } + + /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop + in ZSTD_row_update_internal() at the next search. */ + { + U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); + tagRow[pos] = (BYTE)tag; + row[pos] = ms->nextToUpdate++; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex < curr); + assert(matchIndex >= lowLimit); + + if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { + const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex; + assert(match+4 <= dictEnd); + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dictEnd, prefixStart) + 4; + } + + /* Save best solution */ + if (currentMl > ml) { + ml = currentMl; + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); + if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ + } + } + } + + assert(nbAttempts <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dedicatedDictSearch) { + ml = ZSTD_dedicatedDictSearch_lazy_search(offsetPtr, ml, nbAttempts + ddsExtraAttempts, dms, + ip, iLimit, prefixStart, curr, dictLimit, ddsIdx); + } else if (dictMode == ZSTD_dictMatchState) { + /* TODO: Measure and potentially add prefetching to DMS */ + const U32 dmsLowestIndex = dms->window.dictLimit; + const BYTE* const dmsBase = dms->window.base; + const BYTE* const dmsEnd = dms->window.nextSrc; + const U32 dmsSize = (U32)(dmsEnd - dmsBase); + const U32 dmsIndexDelta = dictLimit - dmsSize; + + { U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth; + U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; + size_t numMatches = 0; + size_t currMatch = 0; + ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries); + + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; + U32 const matchIndex = dmsRow[matchPos]; + if(matchPos == 0) continue; + if (matchIndex < dmsLowestIndex) + break; + PREFETCH_L1(dmsBase + matchIndex); + matchBuffer[numMatches++] = matchIndex; + --nbAttempts; + } + + /* Return the longest match */ + for (; currMatch < numMatches; ++currMatch) { + U32 const matchIndex = matchBuffer[currMatch]; + size_t currentMl=0; + assert(matchIndex >= dmsLowestIndex); + assert(matchIndex < curr); + + { const BYTE* const match = dmsBase + matchIndex; + assert(match+4 <= dmsEnd); + if (MEM_read32(match) == MEM_read32(ip)) + currentMl = ZSTD_count_2segments(ip+4, match+4, iLimit, dmsEnd, prefixStart) + 4; + } + + if (currentMl > ml) { + ml = currentMl; + assert(curr > matchIndex + dmsIndexDelta); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); + if (ip+currentMl == iLimit) break; + } + } + } + } + return ml; +} + + +/** + * Generate search functions templated on (dictMode, mls, rowLog). + * These functions are outlined for code size & compilation time. + * ZSTD_searchMax() dispatches to the correct implementation function. + * + * TODO: The start of the search function involves loading and calculating a + * bunch of constants from the ZSTD_matchState_t. These computations could be + * done in an initialization function, and saved somewhere in the match state. + * Then we could pass a pointer to the saved state instead of the match state, + * and avoid duplicate computations. + * + * TODO: Move the match re-winding into searchMax. This improves compression + * ratio, and unlocks further simplifications with the next TODO. + * + * TODO: Try moving the repcode search into searchMax. After the re-winding + * and repcode search are in searchMax, there is no more logic in the match + * finder loop that requires knowledge about the dictMode. So we should be + * able to avoid force inlining it, and we can join the extDict loop with + * the single segment loop. It should go in searchMax instead of its own + * function to avoid having multiple virtual function calls per search. + */ + +#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls +#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls +#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog + +#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE + +#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offBasePtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offsetPtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \ + return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \ + } \ + +#define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \ + X(dictMode, mls, 4) \ + X(dictMode, mls, 5) \ + X(dictMode, mls, 6) + +#define ZSTD_FOR_EACH_MLS_ROWLOG(X, dictMode) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 4) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 5) \ + ZSTD_FOR_EACH_ROWLOG(X, dictMode, 6) + +#define ZSTD_FOR_EACH_MLS(X, dictMode) \ + X(dictMode, 4) \ + X(dictMode, 5) \ + X(dictMode, 6) + +#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \ + X(__VA_ARGS__, noDict) \ + X(__VA_ARGS__, extDict) \ + X(__VA_ARGS__, dictMatchState) \ + X(__VA_ARGS__, dedicatedDictSearch) + +/* Generate row search fns for each combination of (dictMode, mls, rowLog) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN) +/* Generate binary Tree search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN) +/* Generate hash chain search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN) + +typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e; + +#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + case rowLog: \ + return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr); + +#define ZSTD_SWITCH_MLS(X, dictMode) \ + switch (mls) { \ + ZSTD_FOR_EACH_MLS(X, dictMode) \ + } + +#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \ + case mls: \ + switch (rowLog) { \ + ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \ + } \ + ZSTD_UNREACHABLE; \ + break; + +#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \ + switch (searchMethod) { \ + case search_hashChain: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \ + break; \ + case search_binaryTree: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \ + break; \ + case search_rowHash: \ + ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \ + break; \ + } \ + ZSTD_UNREACHABLE; + +/** + * Searches for the longest match at @p ip. + * Dispatches to the correct implementation function based on the + * (searchMethod, dictMode, mls, rowLog). We use switch statements + * here instead of using an indirect function call through a function + * pointer because after Spectre and Meltdown mitigations, indirect + * function calls can be very costly, especially in the kernel. + * + * NOTE: dictMode and searchMethod should be templated, so those switch + * statements should be optimized out. Only the mls & rowLog switches + * should be left. + * + * @param ms The match state. + * @param ip The position to search at. + * @param iend The end of the input data. + * @param[out] offsetPtr Stores the match offset into this pointer. + * @param mls The minimum search length, in the range [4, 6]. + * @param rowLog The row log (if applicable), in the range [4, 6]. + * @param searchMethod The search method to use (templated). + * @param dictMode The dictMode (templated). + * + * @returns The length of the longest match found, or < mls if no match is found. + * If a match is found its offset is stored in @p offsetPtr. + */ +FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax( + ZSTD_matchState_t* ms, + const BYTE* ip, + const BYTE* iend, + size_t* offsetPtr, + U32 const mls, + U32 const rowLog, + searchMethod_e const searchMethod, + ZSTD_dictMode_e const dictMode) +{ + if (dictMode == ZSTD_noDict) { + ZSTD_SWITCH_SEARCH_METHOD(noDict) + } else if (dictMode == ZSTD_extDict) { + ZSTD_SWITCH_SEARCH_METHOD(extDict) + } else if (dictMode == ZSTD_dictMatchState) { + ZSTD_SWITCH_SEARCH_METHOD(dictMatchState) + } else if (dictMode == ZSTD_dedicatedDictSearch) { + ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch) + } + ZSTD_UNREACHABLE; + return 0; +} + +/* ******************************* +* Common parser - lazy strategy +*********************************/ + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_lazy_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth, + ZSTD_dictMode_e const dictMode) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = (searchMethod == search_rowHash) ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 prefixLowestIndex = ms->window.dictLimit; + const BYTE* const prefixLowest = base + prefixLowestIndex; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + + U32 offset_1 = rep[0], offset_2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const int isDMS = dictMode == ZSTD_dictMatchState; + const int isDDS = dictMode == ZSTD_dedicatedDictSearch; + const int isDxS = isDMS || isDDS; + const ZSTD_matchState_t* const dms = ms->dictMatchState; + const U32 dictLowestIndex = isDxS ? dms->window.dictLimit : 0; + const BYTE* const dictBase = isDxS ? dms->window.base : NULL; + const BYTE* const dictLowest = isDxS ? dictBase + dictLowestIndex : NULL; + const BYTE* const dictEnd = isDxS ? dms->window.nextSrc : NULL; + const U32 dictIndexDelta = isDxS ? + prefixLowestIndex - (U32)(dictEnd - dictBase) : + 0; + const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod); + ip += (dictAndPrefixLength == 0); + if (dictMode == ZSTD_noDict) { + U32 const curr = (U32)(ip - base); + U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); + U32 const maxRep = curr - windowLow; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } + if (isDxS) { + /* dictMatchState repCode checks don't currently handle repCode == 0 + * disabling. */ + assert(offset_1 <= dictAndPrefixLength); + assert(offset_2 <= dictAndPrefixLength); + } + + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offBase = REPCODE1_TO_OFFBASE; + const BYTE* start=ip+1; + DEBUGLOG(7, "search baseline (depth 0)"); + + /* check repCode */ + if (isDxS) { + const U32 repIndex = (U32)(ip - base) + 1 - offset_1; + const BYTE* repMatch = ((dictMode == ZSTD_dictMatchState || dictMode == ZSTD_dedicatedDictSearch) + && repIndex < prefixLowestIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + if (depth==0) goto _storeSequence; + } + } + if ( dictMode == ZSTD_noDict + && ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1)))) { + matchLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + if (depth==0) goto _storeSequence; + } + + /* first search (depth 0) */ + { size_t offbaseFound = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offBase = offbaseFound; + } + + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */; + ip += step; + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + } + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + if (isDxS) { + const U32 repIndex = (U32)(ip - base) - offset_1; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + if (((U32)((prefixLowestIndex-1) - repIndex) >= 3 /* intentional underflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; + size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; + int const gain2 = (int)(mlRep * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((mlRep >= 4) && (gain2 > gain1)) + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; + } + } + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* NOTE: + * Pay attention that `start[-value]` can lead to strange undefined behavior + * notably if `value` is unsigned, resulting in a large positive `-value`. + */ + /* catch up */ + if (OFFBASE_IS_OFFSET(offBase)) { + if (dictMode == ZSTD_noDict) { + while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest)) + && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */ + { start--; matchLength++; } + } + if (isDxS) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); + const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; + const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + } + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); + } + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); + anchor = ip = start + matchLength; + } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } + + /* check immediate repcode */ + if (isDxS) { + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex = current2 - offset_2; + const BYTE* repMatch = repIndex < prefixLowestIndex ? + dictBase - dictIndexDelta + repIndex : + base + repIndex; + if ( ((U32)((prefixLowestIndex-1) - (U32)repIndex) >= 3 /* intentional overflow */) + && (MEM_read32(repMatch) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; + } + break; + } + } + + if (dictMode == ZSTD_noDict) { + while ( ((ip <= ilimit) & (offset_2>0)) + && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { + /* store sequence */ + matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dictMatchState); +} + + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0, ZSTD_dedicatedDictSearch); +} + +/* Row-based matchfinder */ +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_greedy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dictMatchState); +} + + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1, ZSTD_dedicatedDictSearch); +} + +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0, ZSTD_dedicatedDictSearch); +} + +FORCE_INLINE_TEMPLATE +size_t ZSTD_compressBlock_lazy_extDict_generic( + ZSTD_matchState_t* ms, seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const searchMethod_e searchMethod, const U32 depth) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = searchMethod == search_rowHash ? iend - 8 - ZSTD_ROW_HASH_CACHE_SIZE : iend - 8; + const BYTE* const base = ms->window.base; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ms->window.dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ms->window.lowLimit; + const U32 windowLog = ms->cParams.windowLog; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); + + U32 offset_1 = rep[0], offset_2 = rep[1]; + + DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod); + + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + + /* init */ + ip += (ip == prefixStart); + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + + /* Match Loop */ +#if defined(__GNUC__) && defined(__x86_64__) + /* I've measured random a 5% speed loss on levels 5 & 6 (greedy) when the + * code alignment is perturbed. To fix the instability align the loop on 32-bytes. + */ + __asm__(".p2align 5"); +#endif + while (ip < ilimit) { + size_t matchLength=0; + size_t offBase = REPCODE1_TO_OFFBASE; + const BYTE* start=ip+1; + U32 curr = (U32)(ip-base); + + /* check repCode */ + { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr+1, windowLog); + const U32 repIndex = (U32)(curr+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ + & (offset_1 <= curr+1 - windowLow) ) /* note: we are searching at curr+1 */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repEnd, prefixStart) + 4; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offBase = ofbCandidate; + } + + if (matchLength < 4) { + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength); + ip += step + 1; /* jump faster over incompressible sections */ + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; + } } + + /* search match, depth 1 */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_1 <= curr - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + int const gain2 = (int)(repLength * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); + if ((repLength >= 4) && (gain2 > gain1)) + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; + } } + + /* search match, depth 2 */ + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); + if ((ml2 >= 4) && (gain2 > gain1)) { + matchLength = ml2, offBase = ofbCandidate, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (OFFBASE_IS_OFFSET(offBase)) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = (size_t)(start - anchor); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); + anchor = ip = start + matchLength; + } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repCurrent = (U32)(ip-base); + const U32 windowLow = ZSTD_getLowestMatchIndex(ms, repCurrent, windowLog); + const U32 repIndex = repCurrent - offset_2; + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow : do not test positions overlapping 2 memory segments */ + & (offset_2 <= repCurrent - windowLow) ) /* equivalent to `curr > repIndex >= windowLow` */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Save reps for next block */ + rep[0] = offset_1; + rep[1] = offset_2; + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_hashChain, 2); +} + +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_binaryTree, 2); +} + +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 0); +} + +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) + +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 1); +} + +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize) +{ + return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h new file mode 100644 index 000000000..3bde67331 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_lazy.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LAZY_H +#define ZSTD_LAZY_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +/** + * Dedicated Dictionary Search Structure bucket log. In the + * ZSTD_dedicatedDictSearch mode, the hashTable has + * 2 ** ZSTD_LAZY_DDSS_BUCKET_LOG entries in each bucket, rather than just + * one. + */ +#define ZSTD_LAZY_DDSS_BUCKET_LOG 2 + +#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */ + +U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); +void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip); + +void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const BYTE* const ip); + +void ZSTD_preserveUnsortedMark (U32* const table, U32 const size, U32 const reducerValue); /*! used in ZSTD_reduceIndex(). preemptively increase value of ZSTD_DUBT_UNSORTED_MARK */ + +size_t ZSTD_compressBlock_btlazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btlazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dictMatchState_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_dedicatedDictSearch_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_greedy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_greedy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_lazy2_extDict_row( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btlazy2_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_LAZY_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c new file mode 100644 index 000000000..3d74ff19e --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.c @@ -0,0 +1,724 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_ldm.h" + +#include "../common/debug.h" +#include "../common/xxhash.h" +#include "zstd_fast.h" /* ZSTD_fillHashTable() */ +#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */ +#include "zstd_ldm_geartab.h" + +#define LDM_BUCKET_SIZE_LOG 3 +#define LDM_MIN_MATCH_LENGTH 64 +#define LDM_HASH_RLOG 7 + +typedef struct { + U64 rolling; + U64 stopMask; +} ldmRollingHashState_t; + +/** ZSTD_ldm_gear_init(): + * + * Initializes the rolling hash state such that it will honor the + * settings in params. */ +static void ZSTD_ldm_gear_init(ldmRollingHashState_t* state, ldmParams_t const* params) +{ + unsigned maxBitsInMask = MIN(params->minMatchLength, 64); + unsigned hashRateLog = params->hashRateLog; + + state->rolling = ~(U32)0; + + /* The choice of the splitting criterion is subject to two conditions: + * 1. it has to trigger on average every 2^(hashRateLog) bytes; + * 2. ideally, it has to depend on a window of minMatchLength bytes. + * + * In the gear hash algorithm, bit n depends on the last n bytes; + * so in order to obtain a good quality splitting criterion it is + * preferable to use bits with high weight. + * + * To match condition 1 we use a mask with hashRateLog bits set + * and, because of the previous remark, we make sure these bits + * have the highest possible weight while still respecting + * condition 2. + */ + if (hashRateLog > 0 && hashRateLog <= maxBitsInMask) { + state->stopMask = (((U64)1 << hashRateLog) - 1) << (maxBitsInMask - hashRateLog); + } else { + /* In this degenerate case we simply honor the hash rate. */ + state->stopMask = ((U64)1 << hashRateLog) - 1; + } +} + +/** ZSTD_ldm_gear_reset() + * Feeds [data, data + minMatchLength) into the hash without registering any + * splits. This effectively resets the hash state. This is used when skipping + * over data, either at the beginning of a block, or skipping sections. + */ +static void ZSTD_ldm_gear_reset(ldmRollingHashState_t* state, + BYTE const* data, size_t minMatchLength) +{ + U64 hash = state->rolling; + size_t n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + } while (0) + while (n + 3 < minMatchLength) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < minMatchLength) { + GEAR_ITER_ONCE(); + } +#undef GEAR_ITER_ONCE +} + +/** ZSTD_ldm_gear_feed(): + * + * Registers in the splits array all the split points found in the first + * size bytes following the data pointer. This function terminates when + * either all the data has been processed or LDM_BATCH_SIZE splits are + * present in the splits array. + * + * Precondition: The splits array must not be full. + * Returns: The number of bytes processed. */ +static size_t ZSTD_ldm_gear_feed(ldmRollingHashState_t* state, + BYTE const* data, size_t size, + size_t* splits, unsigned* numSplits) +{ + size_t n; + U64 hash, mask; + + hash = state->rolling; + mask = state->stopMask; + n = 0; + +#define GEAR_ITER_ONCE() do { \ + hash = (hash << 1) + ZSTD_ldm_gearTab[data[n] & 0xff]; \ + n += 1; \ + if (UNLIKELY((hash & mask) == 0)) { \ + splits[*numSplits] = n; \ + *numSplits += 1; \ + if (*numSplits == LDM_BATCH_SIZE) \ + goto done; \ + } \ + } while (0) + + while (n + 3 < size) { + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + GEAR_ITER_ONCE(); + } + while (n < size) { + GEAR_ITER_ONCE(); + } + +#undef GEAR_ITER_ONCE + +done: + state->rolling = hash; + return n; +} + +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams) +{ + params->windowLog = cParams->windowLog; + ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX); + DEBUGLOG(4, "ZSTD_ldm_adjustParameters"); + if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG; + if (!params->minMatchLength) params->minMatchLength = LDM_MIN_MATCH_LENGTH; + if (params->hashLog == 0) { + params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); + assert(params->hashLog <= ZSTD_HASHLOG_MAX); + } + if (params->hashRateLog == 0) { + params->hashRateLog = params->windowLog < params->hashLog + ? 0 + : params->windowLog - params->hashLog; + } + params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog); +} + +size_t ZSTD_ldm_getTableSize(ldmParams_t params) +{ + size_t const ldmHSize = ((size_t)1) << params.hashLog; + size_t const ldmBucketSizeLog = MIN(params.bucketSizeLog, params.hashLog); + size_t const ldmBucketSize = ((size_t)1) << (params.hashLog - ldmBucketSizeLog); + size_t const totalSize = ZSTD_cwksp_alloc_size(ldmBucketSize) + + ZSTD_cwksp_alloc_size(ldmHSize * sizeof(ldmEntry_t)); + return params.enableLdm == ZSTD_ps_enable ? totalSize : 0; +} + +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize) +{ + return params.enableLdm == ZSTD_ps_enable ? (maxChunkSize / params.minMatchLength) : 0; +} + +/** ZSTD_ldm_getBucket() : + * Returns a pointer to the start of the bucket associated with hash. */ +static ldmEntry_t* ZSTD_ldm_getBucket( + ldmState_t* ldmState, size_t hash, ldmParams_t const ldmParams) +{ + return ldmState->hashTable + (hash << ldmParams.bucketSizeLog); +} + +/** ZSTD_ldm_insertEntry() : + * Insert the entry with corresponding hash into the hash table */ +static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, + size_t const hash, const ldmEntry_t entry, + ldmParams_t const ldmParams) +{ + BYTE* const pOffset = ldmState->bucketOffsets + hash; + unsigned const offset = *pOffset; + + *(ZSTD_ldm_getBucket(ldmState, hash, ldmParams) + offset) = entry; + *pOffset = (BYTE)((offset + 1) & ((1u << ldmParams.bucketSizeLog) - 1)); + +} + +/** ZSTD_ldm_countBackwardsMatch() : + * Returns the number of bytes that match backwards before pIn and pMatch. + * + * We count only bytes where pMatch >= pBase and pIn >= pAnchor. */ +static size_t ZSTD_ldm_countBackwardsMatch( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase) +{ + size_t matchLength = 0; + while (pIn > pAnchor && pMatch > pMatchBase && pIn[-1] == pMatch[-1]) { + pIn--; + pMatch--; + matchLength++; + } + return matchLength; +} + +/** ZSTD_ldm_countBackwardsMatch_2segments() : + * Returns the number of bytes that match backwards from pMatch, + * even with the backwards match spanning 2 different segments. + * + * On reaching `pMatchBase`, start counting from mEnd */ +static size_t ZSTD_ldm_countBackwardsMatch_2segments( + const BYTE* pIn, const BYTE* pAnchor, + const BYTE* pMatch, const BYTE* pMatchBase, + const BYTE* pExtDictStart, const BYTE* pExtDictEnd) +{ + size_t matchLength = ZSTD_ldm_countBackwardsMatch(pIn, pAnchor, pMatch, pMatchBase); + if (pMatch - matchLength != pMatchBase || pMatchBase == pExtDictStart) { + /* If backwards match is entirely in the extDict or prefix, immediately return */ + return matchLength; + } + DEBUGLOG(7, "ZSTD_ldm_countBackwardsMatch_2segments: found 2-parts backwards match (length in prefix==%zu)", matchLength); + matchLength += ZSTD_ldm_countBackwardsMatch(pIn - matchLength, pAnchor, pExtDictEnd, pExtDictStart); + DEBUGLOG(7, "final backwards match length = %zu", matchLength); + return matchLength; +} + +/** ZSTD_ldm_fillFastTables() : + * + * Fills the relevant tables for the ZSTD_fast and ZSTD_dfast strategies. + * This is similar to ZSTD_loadDictionaryContent. + * + * The tables for the other strategies are filled within their + * block compressors. */ +static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, + void const* end) +{ + const BYTE* const iend = (const BYTE*)end; + + switch(ms->cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); + break; + + case ZSTD_dfast: + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btultra: + case ZSTD_btultra2: + break; + default: + assert(0); /* not possible : not a valid strategy id */ + } + + return 0; +} + +void ZSTD_ldm_fillHashTable( + ldmState_t* ldmState, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params) +{ + U32 const minMatchLength = params->minMatchLength; + U32 const hBits = params->hashLog - params->bucketSizeLog; + BYTE const* const base = ldmState->window.base; + BYTE const* const istart = ip; + ldmRollingHashState_t hashState; + size_t* const splits = ldmState->splitIndices; + unsigned numSplits; + + DEBUGLOG(5, "ZSTD_ldm_fillHashTable"); + + ZSTD_ldm_gear_init(&hashState, params); + while (ip < iend) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, iend - ip, splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + if (ip + splits[n] >= istart + minMatchLength) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + ldmEntry_t entry; + + entry.offset = (U32)(split - base); + entry.checksum = (U32)(xxhash >> 32); + ZSTD_ldm_insertEntry(ldmState, hash, entry, *params); + } + } + + ip += hashed; + } +} + + +/** ZSTD_ldm_limitTableUpdate() : + * + * Sets cctx->nextToUpdate to a position corresponding closer to anchor + * if it is far way + * (after a long match, only update tables a limited amount). */ +static void ZSTD_ldm_limitTableUpdate(ZSTD_matchState_t* ms, const BYTE* anchor) +{ + U32 const curr = (U32)(anchor - ms->window.base); + if (curr > ms->nextToUpdate + 1024) { + ms->nextToUpdate = + curr - MIN(512, curr - ms->nextToUpdate - 1024); + } +} + +static size_t ZSTD_ldm_generateSequences_internal( + ldmState_t* ldmState, rawSeqStore_t* rawSeqStore, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + /* LDM parameters */ + int const extDict = ZSTD_window_hasExtDict(ldmState->window); + U32 const minMatchLength = params->minMatchLength; + U32 const entsPerBucket = 1U << params->bucketSizeLog; + U32 const hBits = params->hashLog - params->bucketSizeLog; + /* Prefix and extDict parameters */ + U32 const dictLimit = ldmState->window.dictLimit; + U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; + BYTE const* const base = ldmState->window.base; + BYTE const* const dictBase = extDict ? ldmState->window.dictBase : NULL; + BYTE const* const dictStart = extDict ? dictBase + lowestIndex : NULL; + BYTE const* const dictEnd = extDict ? dictBase + dictLimit : NULL; + BYTE const* const lowPrefixPtr = base + dictLimit; + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + BYTE const* const ilimit = iend - HASH_READ_SIZE; + /* Input positions */ + BYTE const* anchor = istart; + BYTE const* ip = istart; + /* Rolling hash state */ + ldmRollingHashState_t hashState; + /* Arrays for staged-processing */ + size_t* const splits = ldmState->splitIndices; + ldmMatchCandidate_t* const candidates = ldmState->matchCandidates; + unsigned numSplits; + + if (srcSize < minMatchLength) + return iend - anchor; + + /* Initialize the rolling hash state with the first minMatchLength bytes */ + ZSTD_ldm_gear_init(&hashState, params); + ZSTD_ldm_gear_reset(&hashState, ip, minMatchLength); + ip += minMatchLength; + + while (ip < ilimit) { + size_t hashed; + unsigned n; + + numSplits = 0; + hashed = ZSTD_ldm_gear_feed(&hashState, ip, ilimit - ip, + splits, &numSplits); + + for (n = 0; n < numSplits; n++) { + BYTE const* const split = ip + splits[n] - minMatchLength; + U64 const xxhash = XXH64(split, minMatchLength, 0); + U32 const hash = (U32)(xxhash & (((U32)1 << hBits) - 1)); + + candidates[n].split = split; + candidates[n].hash = hash; + candidates[n].checksum = (U32)(xxhash >> 32); + candidates[n].bucket = ZSTD_ldm_getBucket(ldmState, hash, *params); + PREFETCH_L1(candidates[n].bucket); + } + + for (n = 0; n < numSplits; n++) { + size_t forwardMatchLength = 0, backwardMatchLength = 0, + bestMatchLength = 0, mLength; + U32 offset; + BYTE const* const split = candidates[n].split; + U32 const checksum = candidates[n].checksum; + U32 const hash = candidates[n].hash; + ldmEntry_t* const bucket = candidates[n].bucket; + ldmEntry_t const* cur; + ldmEntry_t const* bestEntry = NULL; + ldmEntry_t newEntry; + + newEntry.offset = (U32)(split - base); + newEntry.checksum = checksum; + + /* If a split point would generate a sequence overlapping with + * the previous one, we merely register it in the hash table and + * move on */ + if (split < anchor) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + for (cur = bucket; cur < bucket + entsPerBucket; cur++) { + size_t curForwardMatchLength, curBackwardMatchLength, + curTotalMatchLength; + if (cur->checksum != checksum || cur->offset <= lowestIndex) { + continue; + } + if (extDict) { + BYTE const* const curMatchBase = + cur->offset < dictLimit ? dictBase : base; + BYTE const* const pMatch = curMatchBase + cur->offset; + BYTE const* const matchEnd = + cur->offset < dictLimit ? dictEnd : iend; + BYTE const* const lowMatchPtr = + cur->offset < dictLimit ? dictStart : lowPrefixPtr; + curForwardMatchLength = + ZSTD_count_2segments(split, pMatch, iend, matchEnd, lowPrefixPtr); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = ZSTD_ldm_countBackwardsMatch_2segments( + split, anchor, pMatch, lowMatchPtr, dictStart, dictEnd); + } else { /* !extDict */ + BYTE const* const pMatch = base + cur->offset; + curForwardMatchLength = ZSTD_count(split, pMatch, iend); + if (curForwardMatchLength < minMatchLength) { + continue; + } + curBackwardMatchLength = + ZSTD_ldm_countBackwardsMatch(split, anchor, pMatch, lowPrefixPtr); + } + curTotalMatchLength = curForwardMatchLength + curBackwardMatchLength; + + if (curTotalMatchLength > bestMatchLength) { + bestMatchLength = curTotalMatchLength; + forwardMatchLength = curForwardMatchLength; + backwardMatchLength = curBackwardMatchLength; + bestEntry = cur; + } + } + + /* No match found -- insert an entry into the hash table + * and process the next candidate match */ + if (bestEntry == NULL) { + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + continue; + } + + /* Match found */ + offset = (U32)(split - base) - bestEntry->offset; + mLength = forwardMatchLength + backwardMatchLength; + { + rawSeq* const seq = rawSeqStore->seq + rawSeqStore->size; + + /* Out of sequence storage */ + if (rawSeqStore->size == rawSeqStore->capacity) + return ERROR(dstSize_tooSmall); + seq->litLength = (U32)(split - backwardMatchLength - anchor); + seq->matchLength = (U32)mLength; + seq->offset = offset; + rawSeqStore->size++; + } + + /* Insert the current entry into the hash table --- it must be + * done after the previous block to avoid clobbering bestEntry */ + ZSTD_ldm_insertEntry(ldmState, hash, newEntry, *params); + + anchor = split + forwardMatchLength; + + /* If we find a match that ends after the data that we've hashed + * then we have a repeating, overlapping, pattern. E.g. all zeros. + * If one repetition of the pattern matches our `stopMask` then all + * repetitions will. We don't need to insert them all into out table, + * only the first one. So skip over overlapping matches. + * This is a major speed boost (20x) for compressing a single byte + * repeated, when that byte ends up in the table. + */ + if (anchor > ip + hashed) { + ZSTD_ldm_gear_reset(&hashState, anchor - minMatchLength, minMatchLength); + /* Continue the outer loop at anchor (ip + hashed == anchor). */ + ip = anchor - hashed; + break; + } + } + + ip += hashed; + } + + return iend - anchor; +} + +/*! ZSTD_ldm_reduceTable() : + * reduce table indexes by `reducerValue` */ +static void ZSTD_ldm_reduceTable(ldmEntry_t* const table, U32 const size, + U32 const reducerValue) +{ + U32 u; + for (u = 0; u < size; u++) { + if (table[u].offset < reducerValue) table[u].offset = 0; + else table[u].offset -= reducerValue; + } +} + +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldmState, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize) +{ + U32 const maxDist = 1U << params->windowLog; + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + size_t const kMaxChunkSize = 1 << 20; + size_t const nbChunks = (srcSize / kMaxChunkSize) + ((srcSize % kMaxChunkSize) != 0); + size_t chunk; + size_t leftoverSize = 0; + + assert(ZSTD_CHUNKSIZE_MAX >= kMaxChunkSize); + /* Check that ZSTD_window_update() has been called for this chunk prior + * to passing it to this function. + */ + assert(ldmState->window.nextSrc >= (BYTE const*)src + srcSize); + /* The input could be very large (in zstdmt), so it must be broken up into + * chunks to enforce the maximum distance and handle overflow correction. + */ + assert(sequences->pos <= sequences->size); + assert(sequences->size <= sequences->capacity); + for (chunk = 0; chunk < nbChunks && sequences->size < sequences->capacity; ++chunk) { + BYTE const* const chunkStart = istart + chunk * kMaxChunkSize; + size_t const remaining = (size_t)(iend - chunkStart); + BYTE const *const chunkEnd = + (remaining < kMaxChunkSize) ? iend : chunkStart + kMaxChunkSize; + size_t const chunkSize = chunkEnd - chunkStart; + size_t newLeftoverSize; + size_t const prevSize = sequences->size; + + assert(chunkStart < iend); + /* 1. Perform overflow correction if necessary. */ + if (ZSTD_window_needOverflowCorrection(ldmState->window, 0, maxDist, ldmState->loadedDictEnd, chunkStart, chunkEnd)) { + U32 const ldmHSize = 1U << params->hashLog; + U32 const correction = ZSTD_window_correctOverflow( + &ldmState->window, /* cycleLog */ 0, maxDist, chunkStart); + ZSTD_ldm_reduceTable(ldmState->hashTable, ldmHSize, correction); + /* invalidate dictionaries on overflow correction */ + ldmState->loadedDictEnd = 0; + } + /* 2. We enforce the maximum offset allowed. + * + * kMaxChunkSize should be small enough that we don't lose too much of + * the window through early invalidation. + * TODO: * Test the chunk size. + * * Try invalidation after the sequence generation and test the + * offset against maxDist directly. + * + * NOTE: Because of dictionaries + sequence splitting we MUST make sure + * that any offset used is valid at the END of the sequence, since it may + * be split into two sequences. This condition holds when using + * ZSTD_window_enforceMaxDist(), but if we move to checking offsets + * against maxDist directly, we'll have to carefully handle that case. + */ + ZSTD_window_enforceMaxDist(&ldmState->window, chunkEnd, maxDist, &ldmState->loadedDictEnd, NULL); + /* 3. Generate the sequences for the chunk, and get newLeftoverSize. */ + newLeftoverSize = ZSTD_ldm_generateSequences_internal( + ldmState, sequences, params, chunkStart, chunkSize); + if (ZSTD_isError(newLeftoverSize)) + return newLeftoverSize; + /* 4. We add the leftover literals from previous iterations to the first + * newly generated sequence, or add the `newLeftoverSize` if none are + * generated. + */ + /* Prepend the leftover literals from the last call */ + if (prevSize < sequences->size) { + sequences->seq[prevSize].litLength += (U32)leftoverSize; + leftoverSize = newLeftoverSize; + } else { + assert(newLeftoverSize == chunkSize); + leftoverSize += chunkSize; + } + } + return 0; +} + +void +ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 const minMatch) +{ + while (srcSize > 0 && rawSeqStore->pos < rawSeqStore->size) { + rawSeq* seq = rawSeqStore->seq + rawSeqStore->pos; + if (srcSize <= seq->litLength) { + /* Skip past srcSize literals */ + seq->litLength -= (U32)srcSize; + return; + } + srcSize -= seq->litLength; + seq->litLength = 0; + if (srcSize < seq->matchLength) { + /* Skip past the first srcSize of the match */ + seq->matchLength -= (U32)srcSize; + if (seq->matchLength < minMatch) { + /* The match is too short, omit it */ + if (rawSeqStore->pos + 1 < rawSeqStore->size) { + seq[1].litLength += seq[0].matchLength; + } + rawSeqStore->pos++; + } + return; + } + srcSize -= seq->matchLength; + seq->matchLength = 0; + rawSeqStore->pos++; + } +} + +/** + * If the sequence length is longer than remaining then the sequence is split + * between this block and the next. + * + * Returns the current sequence to handle, or if the rest of the block should + * be literals, it returns a sequence with offset == 0. + */ +static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore, + U32 const remaining, U32 const minMatch) +{ + rawSeq sequence = rawSeqStore->seq[rawSeqStore->pos]; + assert(sequence.offset > 0); + /* Likely: No partial sequence */ + if (remaining >= sequence.litLength + sequence.matchLength) { + rawSeqStore->pos++; + return sequence; + } + /* Cut the sequence short (offset == 0 ==> rest is literals). */ + if (remaining <= sequence.litLength) { + sequence.offset = 0; + } else if (remaining < sequence.litLength + sequence.matchLength) { + sequence.matchLength = remaining - sequence.litLength; + if (sequence.matchLength < minMatch) { + sequence.offset = 0; + } + } + /* Skip past `remaining` bytes for the future sequences. */ + ZSTD_ldm_skipSequences(rawSeqStore, remaining, minMatch); + return sequence; +} + +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) { + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_paramSwitch_e useRowMatchFinder, + void const* src, size_t srcSize) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + unsigned const minMatch = cParams->minMatch; + ZSTD_blockCompressor const blockCompressor = + ZSTD_selectBlockCompressor(cParams->strategy, useRowMatchFinder, ZSTD_matchState_dictMode(ms)); + /* Input bounds */ + BYTE const* const istart = (BYTE const*)src; + BYTE const* const iend = istart + srcSize; + /* Input positions */ + BYTE const* ip = istart; + + DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize); + /* If using opt parser, use LDMs only as candidates rather than always accepting them */ + if (cParams->strategy >= ZSTD_btopt) { + size_t lastLLSize; + ms->ldmSeqStore = rawSeqStore; + lastLLSize = blockCompressor(ms, seqStore, rep, src, srcSize); + ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore, srcSize); + return lastLLSize; + } + + assert(rawSeqStore->pos <= rawSeqStore->size); + assert(rawSeqStore->size <= rawSeqStore->capacity); + /* Loop through each sequence and apply the block compressor to the literals */ + while (rawSeqStore->pos < rawSeqStore->size && ip < iend) { + /* maybeSplitSequence updates rawSeqStore->pos */ + rawSeq const sequence = maybeSplitSequence(rawSeqStore, + (U32)(iend - ip), minMatch); + int i; + /* End signal */ + if (sequence.offset == 0) + break; + + assert(ip + sequence.litLength + sequence.matchLength <= iend); + + /* Fill tables for block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Run the block compressor */ + DEBUGLOG(5, "pos %u : calling block compressor on segment of size %u", (unsigned)(ip-istart), sequence.litLength); + { + size_t const newLitLength = + blockCompressor(ms, seqStore, rep, ip, sequence.litLength); + ip += sequence.litLength; + /* Update the repcodes */ + for (i = ZSTD_REP_NUM - 1; i > 0; i--) + rep[i] = rep[i-1]; + rep[0] = sequence.offset; + /* Store the sequence */ + ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, + OFFSET_TO_OFFBASE(sequence.offset), + sequence.matchLength); + ip += sequence.matchLength; + } + } + /* Fill the tables for the block compressor */ + ZSTD_ldm_limitTableUpdate(ms, ip); + ZSTD_ldm_fillFastTables(ms, ip); + /* Compress the last literals */ + return blockCompressor(ms, seqStore, rep, ip, iend - ip); +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h new file mode 100644 index 000000000..f147021d2 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_H +#define ZSTD_LDM_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" /* ldmParams_t, U32 */ +#include "../zstd.h" /* ZSTD_CCtx, size_t */ + +/*-************************************* +* Long distance matching +***************************************/ + +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT + +void ZSTD_ldm_fillHashTable( + ldmState_t* state, const BYTE* ip, + const BYTE* iend, ldmParams_t const* params); + +/** + * ZSTD_ldm_generateSequences(): + * + * Generates the sequences using the long distance match finder. + * Generates long range matching sequences in `sequences`, which parse a prefix + * of the source. `sequences` must be large enough to store every sequence, + * which can be checked with `ZSTD_ldm_getMaxNbSeq()`. + * @returns 0 or an error code. + * + * NOTE: The user must have called ZSTD_window_update() for all of the input + * they have, even if they pass it to ZSTD_ldm_generateSequences() in chunks. + * NOTE: This function returns an error if it runs out of space to store + * sequences. + */ +size_t ZSTD_ldm_generateSequences( + ldmState_t* ldms, rawSeqStore_t* sequences, + ldmParams_t const* params, void const* src, size_t srcSize); + +/** + * ZSTD_ldm_blockCompress(): + * + * Compresses a block using the predefined sequences, along with a secondary + * block compressor. The literals section of every sequence is passed to the + * secondary block compressor, and those sequences are interspersed with the + * predefined sequences. Returns the length of the last literals. + * Updates `rawSeqStore.pos` to indicate how many sequences have been consumed. + * `rawSeqStore.seq` may also be updated to split the last sequence between two + * blocks. + * @return The length of the last literals. + * + * NOTE: The source must be at most the maximum block size, but the predefined + * sequences can be any size, and may be longer than the block. In the case that + * they are longer than the block, the last sequences may need to be split into + * two. We handle that case correctly, and update `rawSeqStore` appropriately. + * NOTE: This function does not return any errors. + */ +size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + ZSTD_paramSwitch_e useRowMatchFinder, + void const* src, size_t srcSize); + +/** + * ZSTD_ldm_skipSequences(): + * + * Skip past `srcSize` bytes worth of sequences in `rawSeqStore`. + * Avoids emitting matches less than `minMatch` bytes. + * Must be called for data that is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipSequences(rawSeqStore_t* rawSeqStore, size_t srcSize, + U32 const minMatch); + +/* ZSTD_ldm_skipRawSeqStoreBytes(): + * Moves forward in rawSeqStore by nbBytes, updating fields 'pos' and 'posInSequence'. + * Not to be used in conjunction with ZSTD_ldm_skipSequences(). + * Must be called for data with is not passed to ZSTD_ldm_blockCompress(). + */ +void ZSTD_ldm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes); + +/** ZSTD_ldm_getTableSize() : + * Estimate the space needed for long distance matching tables or 0 if LDM is + * disabled. + */ +size_t ZSTD_ldm_getTableSize(ldmParams_t params); + +/** ZSTD_ldm_getSeqSpace() : + * Return an upper bound on the number of sequences that can be produced by + * the long distance matcher, or 0 if LDM is disabled. + */ +size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); + +/** ZSTD_ldm_adjustParameters() : + * If the params->hashRateLog is not set, set it to its default value based on + * windowLog and params->hashLog. + * + * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to + * params->hashLog if it is not). + * + * Ensures that the minMatchLength >= targetLength during optimal parsing. + */ +void ZSTD_ldm_adjustParameters(ldmParams_t* params, + ZSTD_compressionParameters const* cParams); + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_FAST_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm_geartab.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm_geartab.h new file mode 100644 index 000000000..ef34bc5c9 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_ldm_geartab.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_LDM_GEARTAB_H +#define ZSTD_LDM_GEARTAB_H + +#include "../common/compiler.h" /* UNUSED_ATTR */ +#include "../common/mem.h" /* U64 */ + +static UNUSED_ATTR const U64 ZSTD_ldm_gearTab[256] = { + 0xf5b8f72c5f77775c, 0x84935f266b7ac412, 0xb647ada9ca730ccc, + 0xb065bb4b114fb1de, 0x34584e7e8c3a9fd0, 0x4e97e17c6ae26b05, + 0x3a03d743bc99a604, 0xcecd042422c4044f, 0x76de76c58524259e, + 0x9c8528f65badeaca, 0x86563706e2097529, 0x2902475fa375d889, + 0xafb32a9739a5ebe6, 0xce2714da3883e639, 0x21eaf821722e69e, + 0x37b628620b628, 0x49a8d455d88caf5, 0x8556d711e6958140, + 0x4f7ae74fc605c1f, 0x829f0c3468bd3a20, 0x4ffdc885c625179e, + 0x8473de048a3daf1b, 0x51008822b05646b2, 0x69d75d12b2d1cc5f, + 0x8c9d4a19159154bc, 0xc3cc10f4abbd4003, 0xd06ddc1cecb97391, + 0xbe48e6e7ed80302e, 0x3481db31cee03547, 0xacc3f67cdaa1d210, + 0x65cb771d8c7f96cc, 0x8eb27177055723dd, 0xc789950d44cd94be, + 0x934feadc3700b12b, 0x5e485f11edbdf182, 0x1e2e2a46fd64767a, + 0x2969ca71d82efa7c, 0x9d46e9935ebbba2e, 0xe056b67e05e6822b, + 0x94d73f55739d03a0, 0xcd7010bdb69b5a03, 0x455ef9fcd79b82f4, + 0x869cb54a8749c161, 0x38d1a4fa6185d225, 0xb475166f94bbe9bb, + 0xa4143548720959f1, 0x7aed4780ba6b26ba, 0xd0ce264439e02312, + 0x84366d746078d508, 0xa8ce973c72ed17be, 0x21c323a29a430b01, + 0x9962d617e3af80ee, 0xab0ce91d9c8cf75b, 0x530e8ee6d19a4dbc, + 0x2ef68c0cf53f5d72, 0xc03a681640a85506, 0x496e4e9f9c310967, + 0x78580472b59b14a0, 0x273824c23b388577, 0x66bf923ad45cb553, + 0x47ae1a5a2492ba86, 0x35e304569e229659, 0x4765182a46870b6f, + 0x6cbab625e9099412, 0xddac9a2e598522c1, 0x7172086e666624f2, + 0xdf5003ca503b7837, 0x88c0c1db78563d09, 0x58d51865acfc289d, + 0x177671aec65224f1, 0xfb79d8a241e967d7, 0x2be1e101cad9a49a, + 0x6625682f6e29186b, 0x399553457ac06e50, 0x35dffb4c23abb74, + 0x429db2591f54aade, 0xc52802a8037d1009, 0x6acb27381f0b25f3, + 0xf45e2551ee4f823b, 0x8b0ea2d99580c2f7, 0x3bed519cbcb4e1e1, + 0xff452823dbb010a, 0x9d42ed614f3dd267, 0x5b9313c06257c57b, + 0xa114b8008b5e1442, 0xc1fe311c11c13d4b, 0x66e8763ea34c5568, + 0x8b982af1c262f05d, 0xee8876faaa75fbb7, 0x8a62a4d0d172bb2a, + 0xc13d94a3b7449a97, 0x6dbbba9dc15d037c, 0xc786101f1d92e0f1, + 0xd78681a907a0b79b, 0xf61aaf2962c9abb9, 0x2cfd16fcd3cb7ad9, + 0x868c5b6744624d21, 0x25e650899c74ddd7, 0xba042af4a7c37463, + 0x4eb1a539465a3eca, 0xbe09dbf03b05d5ca, 0x774e5a362b5472ba, + 0x47a1221229d183cd, 0x504b0ca18ef5a2df, 0xdffbdfbde2456eb9, + 0x46cd2b2fbee34634, 0xf2aef8fe819d98c3, 0x357f5276d4599d61, + 0x24a5483879c453e3, 0x88026889192b4b9, 0x28da96671782dbec, + 0x4ef37c40588e9aaa, 0x8837b90651bc9fb3, 0xc164f741d3f0e5d6, + 0xbc135a0a704b70ba, 0x69cd868f7622ada, 0xbc37ba89e0b9c0ab, + 0x47c14a01323552f6, 0x4f00794bacee98bb, 0x7107de7d637a69d5, + 0x88af793bb6f2255e, 0xf3c6466b8799b598, 0xc288c616aa7f3b59, + 0x81ca63cf42fca3fd, 0x88d85ace36a2674b, 0xd056bd3792389e7, + 0xe55c396c4e9dd32d, 0xbefb504571e6c0a6, 0x96ab32115e91e8cc, + 0xbf8acb18de8f38d1, 0x66dae58801672606, 0x833b6017872317fb, + 0xb87c16f2d1c92864, 0xdb766a74e58b669c, 0x89659f85c61417be, + 0xc8daad856011ea0c, 0x76a4b565b6fe7eae, 0xa469d085f6237312, + 0xaaf0365683a3e96c, 0x4dbb746f8424f7b8, 0x638755af4e4acc1, + 0x3d7807f5bde64486, 0x17be6d8f5bbb7639, 0x903f0cd44dc35dc, + 0x67b672eafdf1196c, 0xa676ff93ed4c82f1, 0x521d1004c5053d9d, + 0x37ba9ad09ccc9202, 0x84e54d297aacfb51, 0xa0b4b776a143445, + 0x820d471e20b348e, 0x1874383cb83d46dc, 0x97edeec7a1efe11c, + 0xb330e50b1bdc42aa, 0x1dd91955ce70e032, 0xa514cdb88f2939d5, + 0x2791233fd90db9d3, 0x7b670a4cc50f7a9b, 0x77c07d2a05c6dfa5, + 0xe3778b6646d0a6fa, 0xb39c8eda47b56749, 0x933ed448addbef28, + 0xaf846af6ab7d0bf4, 0xe5af208eb666e49, 0x5e6622f73534cd6a, + 0x297daeca42ef5b6e, 0x862daef3d35539a6, 0xe68722498f8e1ea9, + 0x981c53093dc0d572, 0xfa09b0bfbf86fbf5, 0x30b1e96166219f15, + 0x70e7d466bdc4fb83, 0x5a66736e35f2a8e9, 0xcddb59d2b7c1baef, + 0xd6c7d247d26d8996, 0xea4e39eac8de1ba3, 0x539c8bb19fa3aff2, + 0x9f90e4c5fd508d8, 0xa34e5956fbaf3385, 0x2e2f8e151d3ef375, + 0x173691e9b83faec1, 0xb85a8d56bf016379, 0x8382381267408ae3, + 0xb90f901bbdc0096d, 0x7c6ad32933bcec65, 0x76bb5e2f2c8ad595, + 0x390f851a6cf46d28, 0xc3e6064da1c2da72, 0xc52a0c101cfa5389, + 0xd78eaf84a3fbc530, 0x3781b9e2288b997e, 0x73c2f6dea83d05c4, + 0x4228e364c5b5ed7, 0x9d7a3edf0da43911, 0x8edcfeda24686756, + 0x5e7667a7b7a9b3a1, 0x4c4f389fa143791d, 0xb08bc1023da7cddc, + 0x7ab4be3ae529b1cc, 0x754e6132dbe74ff9, 0x71635442a839df45, + 0x2f6fb1643fbe52de, 0x961e0a42cf7a8177, 0xf3b45d83d89ef2ea, + 0xee3de4cf4a6e3e9b, 0xcd6848542c3295e7, 0xe4cee1664c78662f, + 0x9947548b474c68c4, 0x25d73777a5ed8b0b, 0xc915b1d636b7fc, + 0x21c2ba75d9b0d2da, 0x5f6b5dcf608a64a1, 0xdcf333255ff9570c, + 0x633b922418ced4ee, 0xc136dde0b004b34a, 0x58cc83b05d4b2f5a, + 0x5eb424dda28e42d2, 0x62df47369739cd98, 0xb4e0b42485e4ce17, + 0x16e1f0c1f9a8d1e7, 0x8ec3916707560ebf, 0x62ba6e2df2cc9db3, + 0xcbf9f4ff77d83a16, 0x78d9d7d07d2bbcc4, 0xef554ce1e02c41f4, + 0x8d7581127eccf94d, 0xa9b53336cb3c8a05, 0x38c42c0bf45c4f91, + 0x640893cdf4488863, 0x80ec34bc575ea568, 0x39f324f5b48eaa40, + 0xe9d9ed1f8eff527f, 0x9224fc058cc5a214, 0xbaba00b04cfe7741, + 0x309a9f120fcf52af, 0xa558f3ec65626212, 0x424bec8b7adabe2f, + 0x41622513a6aea433, 0xb88da2d5324ca798, 0xd287733b245528a4, + 0x9a44697e6d68aec3, 0x7b1093be2f49bb28, 0x50bbec632e3d8aad, + 0x6cd90723e1ea8283, 0x897b9e7431b02bf3, 0x219efdcb338a7047, + 0x3b0311f0a27c0656, 0xdb17bf91c0db96e7, 0x8cd4fd6b4e85a5b2, + 0xfab071054ba6409d, 0x40d6fe831fa9dfd9, 0xaf358debad7d791e, + 0xeb8d0e25a65e3e58, 0xbbcbd3df14e08580, 0xcf751f27ecdab2b, + 0x2b4da14f2613d8f4 +}; + +#endif /* ZSTD_LDM_GEARTAB_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.c b/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.c new file mode 100644 index 000000000..f02a76094 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.c @@ -0,0 +1,1472 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "zstd_compress_internal.h" +#include "hist.h" +#include "zstd_opt.h" + + +#define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ +#define ZSTD_MAX_PRICE (1<<30) + +#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ + + +/*-************************************* +* Price functions for optimal parser +***************************************/ + +#if 0 /* approximation at bit level (for tests) */ +# define BITCOST_ACCURACY 0 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat)) +#elif 0 /* fractional bit accuracy (for tests) */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat)) +#else /* opt==approx, ultra==accurate */ +# define BITCOST_ACCURACY 8 +# define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) +# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +#endif + +/* ZSTD_bitWeight() : + * provide estimated "cost" of a stat in full bits only */ +MEM_STATIC U32 ZSTD_bitWeight(U32 stat) +{ + return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); +} + +/* ZSTD_fracWeight() : + * provide fractional-bit "cost" of a stat, + * using linear interpolation approximation */ +MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) +{ + U32 const stat = rawStat + 1; + U32 const hb = ZSTD_highbit32(stat); + U32 const BWeight = hb * BITCOST_MULTIPLIER; + /* Fweight was meant for "Fractional weight" + * but it's effectively a value between 1 and 2 + * using fixed point arithmetic */ + U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; + U32 const weight = BWeight + FWeight; + assert(hb + BITCOST_ACCURACY < 31); + return weight; +} + +#if (DEBUGLEVEL>=2) +/* debugging function, + * @return price in bytes as fractional value + * for debug messages only */ +MEM_STATIC double ZSTD_fCost(int price) +{ + return (double)price / (BITCOST_MULTIPLIER*8); +} +#endif + +static int ZSTD_compressedLiterals(optState_t const* const optPtr) +{ + return optPtr->literalCompressionMode != ZSTD_ps_disable; +} + +static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) +{ + if (ZSTD_compressedLiterals(optPtr)) + optPtr->litSumBasePrice = WEIGHT(optPtr->litSum, optLevel); + optPtr->litLengthSumBasePrice = WEIGHT(optPtr->litLengthSum, optLevel); + optPtr->matchLengthSumBasePrice = WEIGHT(optPtr->matchLengthSum, optLevel); + optPtr->offCodeSumBasePrice = WEIGHT(optPtr->offCodeSum, optLevel); +} + + +static U32 sum_u32(const unsigned table[], size_t nbElts) +{ + size_t n; + U32 total = 0; + for (n=0; n0); + unsigned const newStat = base + (table[s] >> shift); + sum += newStat; + table[s] = newStat; + } + return sum; +} + +/* ZSTD_scaleStats() : + * reduce all elt frequencies in table if sum too large + * return the resulting sum of elements */ +static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) +{ + U32 const prevsum = sum_u32(table, lastEltIndex+1); + U32 const factor = prevsum >> logTarget; + DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); + assert(logTarget < 30); + if (factor <= 1) return prevsum; + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed); +} + +/* ZSTD_rescaleFreqs() : + * if first block (detected by optPtr->litLengthSum == 0) : init statistics + * take hints from dictionary if there is one + * and init from zero if there is none, + * using src for literals stats, and baseline stats for sequence symbols + * otherwise downscale existing stats, to be used as seed for next block. + */ +static void +ZSTD_rescaleFreqs(optState_t* const optPtr, + const BYTE* const src, size_t const srcSize, + int const optLevel) +{ + int const compressedLiterals = ZSTD_compressedLiterals(optPtr); + DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); + optPtr->priceType = zop_dynamic; + + if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */ + + /* heuristic: use pre-defined stats for too small inputs */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { + DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD); + optPtr->priceType = zop_predef; + } + + assert(optPtr->symbolCosts != NULL); + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { + + /* huffman stats covering the full value set : table presumed generated by dictionary */ + optPtr->priceType = zop_dynamic; + + if (compressedLiterals) { + /* generate literals statistics from huffman table */ + unsigned lit; + assert(optPtr->litFreq != NULL); + optPtr->litSum = 0; + for (lit=0; lit<=MaxLit; lit++) { + U32 const scaleLog = 11; /* scale to 2K */ + U32 const bitCost = HUF_getNbBitsFromCTable(optPtr->symbolCosts->huf.CTable, lit); + assert(bitCost <= scaleLog); + optPtr->litFreq[lit] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litSum += optPtr->litFreq[lit]; + } } + + { unsigned ll; + FSE_CState_t llstate; + FSE_initCState(&llstate, optPtr->symbolCosts->fse.litlengthCTable); + optPtr->litLengthSum = 0; + for (ll=0; ll<=MaxLL; ll++) { + U32 const scaleLog = 10; /* scale to 1K */ + U32 const bitCost = FSE_getMaxNbBits(llstate.symbolTT, ll); + assert(bitCost < scaleLog); + optPtr->litLengthFreq[ll] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->litLengthSum += optPtr->litLengthFreq[ll]; + } } + + { unsigned ml; + FSE_CState_t mlstate; + FSE_initCState(&mlstate, optPtr->symbolCosts->fse.matchlengthCTable); + optPtr->matchLengthSum = 0; + for (ml=0; ml<=MaxML; ml++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(mlstate.symbolTT, ml); + assert(bitCost < scaleLog); + optPtr->matchLengthFreq[ml] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->matchLengthSum += optPtr->matchLengthFreq[ml]; + } } + + { unsigned of; + FSE_CState_t ofstate; + FSE_initCState(&ofstate, optPtr->symbolCosts->fse.offcodeCTable); + optPtr->offCodeSum = 0; + for (of=0; of<=MaxOff; of++) { + U32 const scaleLog = 10; + U32 const bitCost = FSE_getMaxNbBits(ofstate.symbolTT, of); + assert(bitCost < scaleLog); + optPtr->offCodeFreq[of] = bitCost ? 1 << (scaleLog-bitCost) : 1 /*minimum to calculate cost*/; + optPtr->offCodeSum += optPtr->offCodeFreq[of]; + } } + + } else { /* first block, no dictionary */ + + assert(optPtr->litFreq != NULL); + if (compressedLiterals) { + /* base initial cost of literals on direct frequency within src */ + unsigned lit = MaxLit; + HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible); + } + + { unsigned const baseLLfreqs[MaxLL+1] = { + 4, 2, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->litLengthFreq, baseLLfreqs, sizeof(baseLLfreqs)); + optPtr->litLengthSum = sum_u32(baseLLfreqs, MaxLL+1); + } + + { unsigned ml; + for (ml=0; ml<=MaxML; ml++) + optPtr->matchLengthFreq[ml] = 1; + } + optPtr->matchLengthSum = MaxML+1; + + { unsigned const baseOFCfreqs[MaxOff+1] = { + 6, 2, 1, 1, 2, 3, 4, 4, + 4, 3, 2, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1 + }; + ZSTD_memcpy(optPtr->offCodeFreq, baseOFCfreqs, sizeof(baseOFCfreqs)); + optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); + } + + } + + } else { /* new block : scale down accumulated statistics */ + + if (compressedLiterals) + optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); + optPtr->litLengthSum = ZSTD_scaleStats(optPtr->litLengthFreq, MaxLL, 11); + optPtr->matchLengthSum = ZSTD_scaleStats(optPtr->matchLengthFreq, MaxML, 11); + optPtr->offCodeSum = ZSTD_scaleStats(optPtr->offCodeFreq, MaxOff, 11); + } + + ZSTD_setBasePrices(optPtr, optLevel); +} + +/* ZSTD_rawLiteralsCost() : + * price of literals (only) in specified segment (which length can be 0). + * does not include price of literalLength symbol */ +static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, + const optState_t* const optPtr, + int optLevel) +{ + if (litLength == 0) return 0; + + if (!ZSTD_compressedLiterals(optPtr)) + return (litLength << 3) * BITCOST_MULTIPLIER; /* Uncompressed - 8 bytes per literal. */ + + if (optPtr->priceType == zop_predef) + return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ + + /* dynamic statistics */ + { U32 price = optPtr->litSumBasePrice * litLength; + U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER; + U32 u; + assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER); + for (u=0; u < litLength; u++) { + U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel); + if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax; + price -= litPrice; + } + return price; + } +} + +/* ZSTD_litLengthPrice() : + * cost of literalLength symbol */ +static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optPtr, int optLevel) +{ + assert(litLength <= ZSTD_BLOCKSIZE_MAX); + if (optPtr->priceType == zop_predef) + return WEIGHT(litLength, optLevel); + + /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX + * because it isn't representable in the zstd format. + * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. + * In such a case, the block would be all literals. + */ + if (litLength == ZSTD_BLOCKSIZE_MAX) + return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel); + + /* dynamic statistics */ + { U32 const llCode = ZSTD_LLcode(litLength); + return (LL_bits[llCode] * BITCOST_MULTIPLIER) + + optPtr->litLengthSumBasePrice + - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); + } +} + +/* ZSTD_getMatchPrice() : + * Provides the cost of the match part (offset + matchLength) of a sequence. + * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. + * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq() + * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_getMatchPrice(U32 const offBase, + U32 const matchLength, + const optState_t* const optPtr, + int const optLevel) +{ + U32 price; + U32 const offCode = ZSTD_highbit32(offBase); + U32 const mlBase = matchLength - MINMATCH; + assert(matchLength >= MINMATCH); + + if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */ + return WEIGHT(mlBase, optLevel) + + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */ + + /* dynamic statistics */ + price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); + if ((optLevel<2) /*static*/ && offCode >= 20) + price += (offCode-19)*2 * BITCOST_MULTIPLIER; /* handicap for long distance offsets, favor decompression speed */ + + /* match Length */ + { U32 const mlCode = ZSTD_MLcode(mlBase); + price += (ML_bits[mlCode] * BITCOST_MULTIPLIER) + (optPtr->matchLengthSumBasePrice - WEIGHT(optPtr->matchLengthFreq[mlCode], optLevel)); + } + + price += BITCOST_MULTIPLIER / 5; /* heuristic : make matches a bit more costly to favor less sequences -> faster decompression speed */ + + DEBUGLOG(8, "ZSTD_getMatchPrice(ml:%u) = %u", matchLength, price); + return price; +} + +/* ZSTD_updateStats() : + * assumption : literals + litLength <= iend */ +static void ZSTD_updateStats(optState_t* const optPtr, + U32 litLength, const BYTE* literals, + U32 offBase, U32 matchLength) +{ + /* literals */ + if (ZSTD_compressedLiterals(optPtr)) { + U32 u; + for (u=0; u < litLength; u++) + optPtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + optPtr->litSum += litLength*ZSTD_LITFREQ_ADD; + } + + /* literal Length */ + { U32 const llCode = ZSTD_LLcode(litLength); + optPtr->litLengthFreq[llCode]++; + optPtr->litLengthSum++; + } + + /* offset code : follows storeSeq() numeric representation */ + { U32 const offCode = ZSTD_highbit32(offBase); + assert(offCode <= MaxOff); + optPtr->offCodeFreq[offCode]++; + optPtr->offCodeSum++; + } + + /* match Length */ + { U32 const mlBase = matchLength - MINMATCH; + U32 const mlCode = ZSTD_MLcode(mlBase); + optPtr->matchLengthFreq[mlCode]++; + optPtr->matchLengthSum++; + } +} + + +/* ZSTD_readMINMATCH() : + * function safe only for comparisons + * assumption : memPtr must be at least 4 bytes before end of buffer */ +MEM_STATIC U32 ZSTD_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (i.e. not within extDict) */ +static U32 ZSTD_insertAndFindFirstIndexHash3 (const ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip) +{ + U32* const hashTable3 = ms->hashTable3; + U32 const hashLog3 = ms->hashLog3; + const BYTE* const base = ms->window.base; + U32 idx = *nextToUpdate3; + U32 const target = (U32)(ip - base); + size_t const hash3 = ZSTD_hash3Ptr(ip, hashLog3); + assert(hashLog3 > 0); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + *nextToUpdate3 = target; + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. + * @param ip assumed <= iend-8 . + * @param target The target of ZSTD_updateTree_internal() - we are filling to this position + * @return : nb of positions added */ +static U32 ZSTD_insertBt1( + const ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + U32 const target, + U32 const mls, const int extDict) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hashLog = cParams->hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = ms->window.base; + const BYTE* const dictBase = ms->window.dictBase; + const U32 dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 curr = (U32)(ip-base); + const U32 btLow = btMask >= curr ? 0 : curr - btMask; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + /* windowLow is based on target because + * we only need positions that will be in the window at the end of the tree update. + */ + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, target, cParams->windowLog); + U32 matchEndIdx = curr+8+1; + size_t bestLength = 8; + U32 nbCompares = 1U << cParams->searchLog; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((curr-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((curr-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ + + DEBUGLOG(8, "ZSTD_insertBt1 (%u)", curr); + + assert(curr <= target); + assert(ip <= iend-8); /* required for h calculation */ + hashTable[h] = curr; /* Update Hash Table */ + + assert(windowLow > 0); + for (; nbCompares && (matchIndex >= windowLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(matchIndex < curr); + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + + if (!extDict || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* might be wrong if actually extDict */ + match = base + matchIndex; + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iend); + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt tree */ + } + + if (match[matchLength] < ip[matchLength]) { /* necessarily within buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + smallerPtr = nextPtr+1; /* new "candidate" => larger than match, which was smaller than target */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous and closer to current */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop searching */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + { U32 positions = 0; + if (bestLength > 384) positions = MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + assert(matchEndIdx > curr + 8); + return MAX(positions, matchEndIdx - (curr + 8)); + } +} + +FORCE_INLINE_TEMPLATE +void ZSTD_updateTree_internal( + ZSTD_matchState_t* ms, + const BYTE* const ip, const BYTE* const iend, + const U32 mls, const ZSTD_dictMode_e dictMode) +{ + const BYTE* const base = ms->window.base; + U32 const target = (U32)(ip - base); + U32 idx = ms->nextToUpdate; + DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + idx, target, dictMode); + + while(idx < target) { + U32 const forward = ZSTD_insertBt1(ms, base+idx, iend, target, mls, dictMode == ZSTD_extDict); + assert(idx < (U32)(idx + forward)); + idx += forward; + } + assert((size_t)(ip - base) <= (size_t)(U32)(-1)); + assert((size_t)(iend - base) <= (size_t)(U32)(-1)); + ms->nextToUpdate = target; +} + +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { + ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); +} + +FORCE_INLINE_TEMPLATE U32 +ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip, const BYTE* const iLimit, + const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + const U32 lengthToBeat, + const U32 mls /* template */) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + const BYTE* const base = ms->window.base; + U32 const curr = (U32)(ip-base); + U32 const hashLog = cParams->hashLog; + U32 const minMatch = (mls==3) ? 3 : 4; + U32* const hashTable = ms->hashTable; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32 matchIndex = hashTable[h]; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = ms->window.dictBase; + U32 const dictLimit = ms->window.dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + U32 const btLow = (btMask >= curr) ? 0 : curr - btMask; + U32 const windowLow = ZSTD_getLowestMatchIndex(ms, curr, cParams->windowLog); + U32 const matchLow = windowLow ? windowLow : 1; + U32* smallerPtr = bt + 2*(curr&btMask); + U32* largerPtr = bt + 2*(curr&btMask) + 1; + U32 matchEndIdx = curr+8+1; /* farthest referenced position of any match => detects repetitive patterns */ + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + U32 nbCompares = 1U << cParams->searchLog; + + const ZSTD_matchState_t* dms = dictMode == ZSTD_dictMatchState ? ms->dictMatchState : NULL; + const ZSTD_compressionParameters* const dmsCParams = + dictMode == ZSTD_dictMatchState ? &dms->cParams : NULL; + const BYTE* const dmsBase = dictMode == ZSTD_dictMatchState ? dms->window.base : NULL; + const BYTE* const dmsEnd = dictMode == ZSTD_dictMatchState ? dms->window.nextSrc : NULL; + U32 const dmsHighLimit = dictMode == ZSTD_dictMatchState ? (U32)(dmsEnd - dmsBase) : 0; + U32 const dmsLowLimit = dictMode == ZSTD_dictMatchState ? dms->window.lowLimit : 0; + U32 const dmsIndexDelta = dictMode == ZSTD_dictMatchState ? windowLow - dmsHighLimit : 0; + U32 const dmsHashLog = dictMode == ZSTD_dictMatchState ? dmsCParams->hashLog : hashLog; + U32 const dmsBtLog = dictMode == ZSTD_dictMatchState ? dmsCParams->chainLog - 1 : btLog; + U32 const dmsBtMask = dictMode == ZSTD_dictMatchState ? (1U << dmsBtLog) - 1 : 0; + U32 const dmsBtLow = dictMode == ZSTD_dictMatchState && dmsBtMask < dmsHighLimit - dmsLowLimit ? dmsHighLimit - dmsBtMask : dmsLowLimit; + + size_t bestLength = lengthToBeat-1; + DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", curr); + + /* check repCode */ + assert(ll0 <= 1); /* necessarily 1 or 0 */ + { U32 const lastR = ZSTD_REP_NUM + ll0; + U32 repCode; + for (repCode = ll0; repCode < lastR; repCode++) { + U32 const repOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; + U32 const repIndex = curr - repOffset; + U32 repLen = 0; + assert(curr >= dictLimit); + if (repOffset-1 /* intentional overflow, discards 0 and -1 */ < curr-dictLimit) { /* equivalent to `curr > repIndex >= dictLimit` */ + /* We must validate the repcode offset because when we're using a dictionary the + * valid offset range shrinks when the dictionary goes out of bounds. + */ + if ((repIndex >= windowLow) & (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(ip - repOffset, minMatch))) { + repLen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repOffset, iLimit) + minMatch; + } + } else { /* repIndex < dictLimit || repIndex >= curr */ + const BYTE* const repMatch = dictMode == ZSTD_dictMatchState ? + dmsBase + repIndex - dmsIndexDelta : + dictBase + repIndex; + assert(curr >= windowLow); + if ( dictMode == ZSTD_extDict + && ( ((repOffset-1) /*intentional overflow*/ < curr - windowLow) /* equivalent to `curr > repIndex >= windowLow` */ + & (((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */) + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dictEnd, prefixStart) + minMatch; + } + if (dictMode == ZSTD_dictMatchState + && ( ((repOffset-1) /*intentional overflow*/ < curr - (dmsLowLimit + dmsIndexDelta)) /* equivalent to `curr > repIndex >= dmsLowLimit` */ + & ((U32)((dictLimit-1) - repIndex) >= 3) ) /* intentional overflow : do not test positions overlapping 2 memory segments */ + && (ZSTD_readMINMATCH(ip, minMatch) == ZSTD_readMINMATCH(repMatch, minMatch)) ) { + repLen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iLimit, dmsEnd, prefixStart) + minMatch; + } } + /* save longer solution */ + if (repLen > bestLength) { + DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", + repCode, ll0, repOffset, repLen); + bestLength = repLen; + matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */ + matches[mnum].len = (U32)repLen; + mnum++; + if ( (repLen > sufficient_len) + | (ip+repLen == iLimit) ) { /* best possible */ + return mnum; + } } } } + + /* HC3 match finder */ + if ((mls == 3) /*static*/ && (bestLength < mls)) { + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3(ms, nextToUpdate3, ip); + if ((matchIndex3 >= matchLow) + & (curr - matchIndex3 < (1<<18)) /*heuristic : longer distance likely too expensive*/ ) { + size_t mlen; + if ((dictMode == ZSTD_noDict) /*static*/ || (dictMode == ZSTD_dictMatchState) /*static*/ || (matchIndex3 >= dictLimit)) { + const BYTE* const match = base + matchIndex3; + mlen = ZSTD_count(ip, match, iLimit); + } else { + const BYTE* const match = dictBase + matchIndex3; + mlen = ZSTD_count_2segments(ip, match, iLimit, dictEnd, prefixStart); + } + + /* save best solution */ + if (mlen >= mls /* == 3 > bestLength */) { + DEBUGLOG(8, "found small match with hlog3, of length %u", + (U32)mlen); + bestLength = mlen; + assert(curr > matchIndex3); + assert(mnum==0); /* no prior solution */ + matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3); + matches[0].len = (U32)mlen; + mnum = 1; + if ( (mlen > sufficient_len) | + (ip+mlen == iLimit) ) { /* best possible length */ + ms->nextToUpdate = curr+1; /* skip insertion */ + return 1; + } } } + /* no dictMatchState lookup: dicts don't have a populated HC3 table */ + } /* if (mls == 3) */ + + hashTable[h] = curr; /* Update Hash Table */ + + for (; nbCompares && (matchIndex >= matchLow); --nbCompares) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + const BYTE* match; + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + assert(curr > matchIndex); + + if ((dictMode == ZSTD_noDict) || (dictMode == ZSTD_dictMatchState) || (matchIndex+matchLength >= dictLimit)) { + assert(matchIndex+matchLength >= dictLimit); /* ensure the condition is correct when !extDict */ + match = base + matchIndex; + if (matchIndex >= dictLimit) assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count(ip+matchLength, match+matchLength, iLimit); + } else { + match = dictBase + matchIndex; + assert(memcmp(match, ip, matchLength) == 0); /* ensure early section of match is equal as expected */ + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* prepare for match[matchLength] read */ + } + + if (matchLength > bestLength) { + DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); + assert(matchEndIdx > matchIndex); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + if (dictMode == ZSTD_dictMatchState) nbCompares = 0; /* break should also skip searching dms */ + break; /* drop, to preserve bt consistency (miss a little bit of compression) */ + } } + + if (match[matchLength] < ip[matchLength]) { + /* match smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new candidate => larger than match, which was smaller than current */ + matchIndex = nextPtr[1]; /* new matchIndex, larger than previous, closer to current */ + } else { + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + assert(nbCompares <= (1U << ZSTD_SEARCHLOG_MAX)); /* Check we haven't underflowed. */ + if (dictMode == ZSTD_dictMatchState && nbCompares) { + size_t const dmsH = ZSTD_hashPtr(ip, dmsHashLog, mls); + U32 dictMatchIndex = dms->hashTable[dmsH]; + const U32* const dmsBt = dms->chainTable; + commonLengthSmaller = commonLengthLarger = 0; + for (; nbCompares && (dictMatchIndex > dmsLowLimit); --nbCompares) { + const U32* const nextPtr = dmsBt + 2*(dictMatchIndex & dmsBtMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match = dmsBase + dictMatchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dmsEnd, prefixStart); + if (dictMatchIndex+matchLength >= dmsHighLimit) + match = base + dictMatchIndex + dmsIndexDelta; /* to prepare for next usage of match[matchLength] */ + + if (matchLength > bestLength) { + matchIndex = dictMatchIndex + dmsIndexDelta; + DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); + matches[mnum].len = (U32)matchLength; + mnum++; + if ( (matchLength > ZSTD_OPT_NUM) + | (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */) { + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } } + + if (dictMatchIndex <= dmsBtLow) { break; } /* beyond tree size, stop the search */ + if (match[matchLength] < ip[matchLength]) { + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + dictMatchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + commonLengthLarger = matchLength; + dictMatchIndex = nextPtr[0]; + } } } /* if (dictMode == ZSTD_dictMatchState) */ + + assert(matchEndIdx > curr+8); + ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ + return mnum; +} + +typedef U32 (*ZSTD_getAllMatchesFn)( + ZSTD_match_t*, + ZSTD_matchState_t*, + U32*, + const BYTE*, + const BYTE*, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat); + +FORCE_INLINE_TEMPLATE U32 ZSTD_btGetAllMatches_internal( + ZSTD_match_t* matches, + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* ip, + const BYTE* const iHighLimit, + const U32 rep[ZSTD_REP_NUM], + U32 const ll0, + U32 const lengthToBeat, + const ZSTD_dictMode_e dictMode, + const U32 mls) +{ + assert(BOUNDED(3, ms->cParams.minMatch, 6) == mls); + DEBUGLOG(8, "ZSTD_BtGetAllMatches(dictMode=%d, mls=%u)", (int)dictMode, mls); + if (ip < ms->window.base + ms->nextToUpdate) + return 0; /* skipped area */ + ZSTD_updateTree_internal(ms, ip, iHighLimit, mls, dictMode); + return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, mls); +} + +#define ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls) ZSTD_btGetAllMatches_##dictMode##_##mls + +#define GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, mls) \ + static U32 ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, mls)( \ + ZSTD_match_t* matches, \ + ZSTD_matchState_t* ms, \ + U32* nextToUpdate3, \ + const BYTE* ip, \ + const BYTE* const iHighLimit, \ + const U32 rep[ZSTD_REP_NUM], \ + U32 const ll0, \ + U32 const lengthToBeat) \ + { \ + return ZSTD_btGetAllMatches_internal( \ + matches, ms, nextToUpdate3, ip, iHighLimit, \ + rep, ll0, lengthToBeat, ZSTD_##dictMode, mls); \ + } + +#define GEN_ZSTD_BT_GET_ALL_MATCHES(dictMode) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 3) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 4) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 5) \ + GEN_ZSTD_BT_GET_ALL_MATCHES_(dictMode, 6) + +GEN_ZSTD_BT_GET_ALL_MATCHES(noDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(extDict) +GEN_ZSTD_BT_GET_ALL_MATCHES(dictMatchState) + +#define ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMode) \ + { \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 3), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 4), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 5), \ + ZSTD_BT_GET_ALL_MATCHES_FN(dictMode, 6) \ + } + +static ZSTD_getAllMatchesFn +ZSTD_selectBtGetAllMatches(ZSTD_matchState_t const* ms, ZSTD_dictMode_e const dictMode) +{ + ZSTD_getAllMatchesFn const getAllMatchesFns[3][4] = { + ZSTD_BT_GET_ALL_MATCHES_ARRAY(noDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(extDict), + ZSTD_BT_GET_ALL_MATCHES_ARRAY(dictMatchState) + }; + U32 const mls = BOUNDED(3, ms->cParams.minMatch, 6); + assert((U32)dictMode < 3); + assert(mls - 3 < 4); + return getAllMatchesFns[(int)dictMode][mls - 3]; +} + +/************************* +* LDM helper functions * +*************************/ + +/* Struct containing info needed to make decision about ldm inclusion */ +typedef struct { + rawSeqStore_t seqStore; /* External match candidates store for this block */ + U32 startPosInBlock; /* Start position of the current match candidate */ + U32 endPosInBlock; /* End position of the current match candidate */ + U32 offset; /* Offset of the match candidate */ +} ZSTD_optLdm_t; + +/* ZSTD_optLdm_skipRawSeqStoreBytes(): + * Moves forward in @rawSeqStore by @nbBytes, + * which will update the fields 'pos' and 'posInSequence'. + */ +static void ZSTD_optLdm_skipRawSeqStoreBytes(rawSeqStore_t* rawSeqStore, size_t nbBytes) +{ + U32 currPos = (U32)(rawSeqStore->posInSequence + nbBytes); + while (currPos && rawSeqStore->pos < rawSeqStore->size) { + rawSeq currSeq = rawSeqStore->seq[rawSeqStore->pos]; + if (currPos >= currSeq.litLength + currSeq.matchLength) { + currPos -= currSeq.litLength + currSeq.matchLength; + rawSeqStore->pos++; + } else { + rawSeqStore->posInSequence = currPos; + break; + } + } + if (currPos == 0 || rawSeqStore->pos == rawSeqStore->size) { + rawSeqStore->posInSequence = 0; + } +} + +/* ZSTD_opt_getNextMatchAndUpdateSeqStore(): + * Calculates the beginning and end of the next match in the current block. + * Updates 'pos' and 'posInSequence' of the ldmSeqStore. + */ +static void +ZSTD_opt_getNextMatchAndUpdateSeqStore(ZSTD_optLdm_t* optLdm, U32 currPosInBlock, + U32 blockBytesRemaining) +{ + rawSeq currSeq; + U32 currBlockEndPos; + U32 literalsBytesRemaining; + U32 matchBytesRemaining; + + /* Setting match end position to MAX to ensure we never use an LDM during this block */ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + return; + } + /* Calculate appropriate bytes left in matchLength and litLength + * after adjusting based on ldmSeqStore->posInSequence */ + currSeq = optLdm->seqStore.seq[optLdm->seqStore.pos]; + assert(optLdm->seqStore.posInSequence <= currSeq.litLength + currSeq.matchLength); + currBlockEndPos = currPosInBlock + blockBytesRemaining; + literalsBytesRemaining = (optLdm->seqStore.posInSequence < currSeq.litLength) ? + currSeq.litLength - (U32)optLdm->seqStore.posInSequence : + 0; + matchBytesRemaining = (literalsBytesRemaining == 0) ? + currSeq.matchLength - ((U32)optLdm->seqStore.posInSequence - currSeq.litLength) : + currSeq.matchLength; + + /* If there are more literal bytes than bytes remaining in block, no ldm is possible */ + if (literalsBytesRemaining >= blockBytesRemaining) { + optLdm->startPosInBlock = UINT_MAX; + optLdm->endPosInBlock = UINT_MAX; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, blockBytesRemaining); + return; + } + + /* Matches may be < MINMATCH by this process. In that case, we will reject them + when we are deciding whether or not to add the ldm */ + optLdm->startPosInBlock = currPosInBlock + literalsBytesRemaining; + optLdm->endPosInBlock = optLdm->startPosInBlock + matchBytesRemaining; + optLdm->offset = currSeq.offset; + + if (optLdm->endPosInBlock > currBlockEndPos) { + /* Match ends after the block ends, we can't use the whole match */ + optLdm->endPosInBlock = currBlockEndPos; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, currBlockEndPos - currPosInBlock); + } else { + /* Consume nb of bytes equal to size of sequence left */ + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, literalsBytesRemaining + matchBytesRemaining); + } +} + +/* ZSTD_optLdm_maybeAddMatch(): + * Adds a match if it's long enough, + * based on it's 'matchStartPosInBlock' and 'matchEndPosInBlock', + * into 'matches'. Maintains the correct ordering of 'matches'. + */ +static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, + const ZSTD_optLdm_t* optLdm, U32 currPosInBlock) +{ + U32 const posDiff = currPosInBlock - optLdm->startPosInBlock; + /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */ + U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; + + /* Ensure that current block position is not outside of the match */ + if (currPosInBlock < optLdm->startPosInBlock + || currPosInBlock >= optLdm->endPosInBlock + || candidateMatchLength < MINMATCH) { + return; + } + + if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { + U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset); + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u", + candidateOffBase, candidateMatchLength, currPosInBlock); + matches[*nbMatches].len = candidateMatchLength; + matches[*nbMatches].off = candidateOffBase; + (*nbMatches)++; + } +} + +/* ZSTD_optLdm_processMatchCandidate(): + * Wrapper function to update ldm seq store and call ldm functions as necessary. + */ +static void +ZSTD_optLdm_processMatchCandidate(ZSTD_optLdm_t* optLdm, + ZSTD_match_t* matches, U32* nbMatches, + U32 currPosInBlock, U32 remainingBytes) +{ + if (optLdm->seqStore.size == 0 || optLdm->seqStore.pos >= optLdm->seqStore.size) { + return; + } + + if (currPosInBlock >= optLdm->endPosInBlock) { + if (currPosInBlock > optLdm->endPosInBlock) { + /* The position at which ZSTD_optLdm_processMatchCandidate() is called is not necessarily + * at the end of a match from the ldm seq store, and will often be some bytes + * over beyond matchEndPosInBlock. As such, we need to correct for these "overshoots" + */ + U32 const posOvershoot = currPosInBlock - optLdm->endPosInBlock; + ZSTD_optLdm_skipRawSeqStoreBytes(&optLdm->seqStore, posOvershoot); + } + ZSTD_opt_getNextMatchAndUpdateSeqStore(optLdm, currPosInBlock, remainingBytes); + } + ZSTD_optLdm_maybeAddMatch(matches, nbMatches, optLdm, currPosInBlock); +} + + +/*-******************************* +* Optimal parser +*********************************/ + +static U32 ZSTD_totalLen(ZSTD_optimal_t sol) +{ + return sol.litlen + sol.mlen; +} + +#if 0 /* debug */ + +static void +listStats(const U32* table, int lastEltID) +{ + int const nbElts = lastEltID + 1; + int enb; + for (enb=0; enb < nbElts; enb++) { + (void)table; + /* RAWLOG(2, "%3i:%3i, ", enb, table[enb]); */ + RAWLOG(2, "%4i,", table[enb]); + } + RAWLOG(2, " \n"); +} + +#endif + +FORCE_INLINE_TEMPLATE size_t +ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, + const int optLevel, + const ZSTD_dictMode_e dictMode) +{ + optState_t* const optStatePtr = &ms->opt; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ms->window.base; + const BYTE* const prefixStart = base + ms->window.dictLimit; + const ZSTD_compressionParameters* const cParams = &ms->cParams; + + ZSTD_getAllMatchesFn getAllMatches = ZSTD_selectBtGetAllMatches(ms, dictMode); + + U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); + U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; + U32 nextToUpdate3 = ms->nextToUpdate; + + ZSTD_optimal_t* const opt = optStatePtr->priceTable; + ZSTD_match_t* const matches = optStatePtr->matchTable; + ZSTD_optimal_t lastSequence; + ZSTD_optLdm_t optLdm; + + ZSTD_memset(&lastSequence, 0, sizeof(ZSTD_optimal_t)); + + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; + optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; + ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); + + /* init */ + DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", + (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); + assert(optLevel <= 2); + ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); + ip += (ip==prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, last_pos = 0; + + /* find first match */ + { U32 const litlen = (U32)(ip - anchor); + U32 const ll0 = !litlen; + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, ip, iend, rep, ll0, minMatch); + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(ip-istart), (U32)(iend - ip)); + if (!nbMatches) { ip++; continue; } + + /* initialize opt[0] */ + { U32 i ; for (i=0; i immediate encoding */ + { U32 const maxML = matches[nbMatches-1].len; + U32 const maxOffBase = matches[nbMatches-1].off; + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series", + nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart)); + + if (maxML > sufficient_len) { + lastSequence.litlen = litlen; + lastSequence.mlen = maxML; + lastSequence.off = maxOffBase; + DEBUGLOG(6, "large match (%u>%u), immediate encoding", + maxML, sufficient_len); + cur = 0; + last_pos = ZSTD_totalLen(lastSequence); + goto _shortestPath; + } } + + /* set prices for first matches starting position == 0 */ + assert(opt[0].price >= 0); + { U32 const literalsPrice = (U32)opt[0].price + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 pos; + U32 matchNb; + for (pos = 1; pos < minMatch; pos++) { + opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ + } + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offBase = matches[matchNb].off; + U32 const end = matches[matchNb].len; + for ( ; pos <= end ; pos++ ) { + U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel); + U32 const sequencePrice = literalsPrice + matchPrice; + DEBUGLOG(7, "rPos:%u => set initial price : %.2f", + pos, ZSTD_fCost((int)sequencePrice)); + opt[pos].mlen = pos; + opt[pos].off = offBase; + opt[pos].litlen = litlen; + opt[pos].price = (int)sequencePrice; + } } + last_pos = pos-1; + } + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + const BYTE* const inr = ip + cur; + assert(cur < ZSTD_OPT_NUM); + DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur) + + /* Fix current position with one literal if cheaper */ + { U32 const litlen = (opt[cur-1].mlen == 0) ? opt[cur-1].litlen + 1 : 1; + int const price = opt[cur-1].price + + (int)ZSTD_rawLiteralsCost(ip+cur-1, 1, optStatePtr, optLevel) + + (int)ZSTD_litLengthPrice(litlen, optStatePtr, optLevel) + - (int)ZSTD_litLengthPrice(litlen-1, optStatePtr, optLevel); + assert(price < 1000000000); /* overflow check */ + if (price <= opt[cur].price) { + DEBUGLOG(7, "cPos:%zi==rPos:%u : better price (%.2f<=%.2f) using literal (ll==%u) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), litlen, + opt[cur-1].rep[0], opt[cur-1].rep[1], opt[cur-1].rep[2]); + opt[cur].mlen = 0; + opt[cur].off = 0; + opt[cur].litlen = litlen; + opt[cur].price = price; + } else { + DEBUGLOG(7, "cPos:%zi==rPos:%u : literal would cost more (%.2f>%.2f) (hist:%u,%u,%u)", + inr-istart, cur, ZSTD_fCost(price), ZSTD_fCost(opt[cur].price), + opt[cur].rep[0], opt[cur].rep[1], opt[cur].rep[2]); + } + } + + /* Set the repcodes of the current position. We must do it here + * because we rely on the repcodes of the 2nd to last sequence being + * correct to set the next chunks repcodes during the backward + * traversal. + */ + ZSTD_STATIC_ASSERT(sizeof(opt[cur].rep) == sizeof(repcodes_t)); + assert(cur >= opt[cur].mlen); + if (opt[cur].mlen != 0) { + U32 const prev = cur - opt[cur].mlen; + repcodes_t const newReps = ZSTD_newRep(opt[prev].rep, opt[cur].off, opt[cur].litlen==0); + ZSTD_memcpy(opt[cur].rep, &newReps, sizeof(repcodes_t)); + } else { + ZSTD_memcpy(opt[cur].rep, opt[cur - 1].rep, sizeof(repcodes_t)); + } + + /* last match must start at a minimum distance of 8 from oend */ + if (inr > ilimit) continue; + + if (cur == last_pos) break; + + if ( (optLevel==0) /*static_test*/ + && (opt[cur+1].price <= opt[cur].price + (BITCOST_MULTIPLIER/2)) ) { + DEBUGLOG(7, "move to next rPos:%u : price is <=", cur+1); + continue; /* skip unpromising positions; about ~+6% speed, -0.01 ratio */ + } + + assert(opt[cur].price >= 0); + { U32 const ll0 = (opt[cur].mlen != 0); + U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0; + U32 const previousPrice = (U32)opt[cur].price; + U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel); + U32 nbMatches = getAllMatches(matches, ms, &nextToUpdate3, inr, iend, opt[cur].rep, ll0, minMatch); + U32 matchNb; + + ZSTD_optLdm_processMatchCandidate(&optLdm, matches, &nbMatches, + (U32)(inr-istart), (U32)(iend-inr)); + + if (!nbMatches) { + DEBUGLOG(7, "rPos:%u : no match found", cur); + continue; + } + + { U32 const maxML = matches[nbMatches-1].len; + DEBUGLOG(7, "cPos:%zi==rPos:%u, found %u matches, of maxLength=%u", + inr-istart, cur, nbMatches, maxML); + + if ( (maxML > sufficient_len) + || (cur + maxML >= ZSTD_OPT_NUM) ) { + lastSequence.mlen = maxML; + lastSequence.off = matches[nbMatches-1].off; + lastSequence.litlen = litlen; + cur -= (opt[cur].mlen==0) ? opt[cur].litlen : 0; /* last sequence is actually only literals, fix cur to last match - note : may underflow, in which case, it's first sequence, and it's okay */ + last_pos = cur + ZSTD_totalLen(lastSequence); + if (cur > ZSTD_OPT_NUM) cur = 0; /* underflow => first match */ + goto _shortestPath; + } } + + /* set prices using matches found at position == cur */ + for (matchNb = 0; matchNb < nbMatches; matchNb++) { + U32 const offset = matches[matchNb].off; + U32 const lastML = matches[matchNb].len; + U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; + U32 mlen; + + DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u", + matchNb, matches[matchNb].off, lastML, litlen); + + for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ + U32 const pos = cur + mlen; + int const price = (int)basePrice + (int)ZSTD_getMatchPrice(offset, mlen, optStatePtr, optLevel); + + if ((pos > last_pos) || (price < opt[pos].price)) { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new better price (%.2f<%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } /* fill empty positions */ + opt[pos].mlen = mlen; + opt[pos].off = offset; + opt[pos].litlen = litlen; + opt[pos].price = price; + } else { + DEBUGLOG(7, "rPos:%u (ml=%2u) => new price is worse (%.2f>=%.2f)", + pos, mlen, ZSTD_fCost(price), ZSTD_fCost(opt[pos].price)); + if (optLevel==0) break; /* early update abort; gets ~+10% speed for about -0.01 ratio loss */ + } + } } } + } /* for (cur = 1; cur <= last_pos; cur++) */ + + lastSequence = opt[last_pos]; + cur = last_pos > ZSTD_totalLen(lastSequence) ? last_pos - ZSTD_totalLen(lastSequence) : 0; /* single sequence, and it starts before `ip` */ + assert(cur < ZSTD_OPT_NUM); /* control overflow*/ + +_shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ + assert(opt[0].mlen == 0); + + /* Set the next chunk's repcodes based on the repcodes of the beginning + * of the last match, and the last sequence. This avoids us having to + * update them while traversing the sequences. + */ + if (lastSequence.mlen != 0) { + repcodes_t const reps = ZSTD_newRep(opt[cur].rep, lastSequence.off, lastSequence.litlen==0); + ZSTD_memcpy(rep, &reps, sizeof(reps)); + } else { + ZSTD_memcpy(rep, opt[cur].rep, sizeof(repcodes_t)); + } + + { U32 const storeEnd = cur + 1; + U32 storeStart = storeEnd; + U32 seqPos = cur; + + DEBUGLOG(6, "start reverse traversal (last_pos:%u, cur:%u)", + last_pos, cur); (void)last_pos; + assert(storeEnd < ZSTD_OPT_NUM); + DEBUGLOG(6, "last sequence copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + storeEnd, lastSequence.litlen, lastSequence.mlen, lastSequence.off); + opt[storeEnd] = lastSequence; + while (seqPos > 0) { + U32 const backDist = ZSTD_totalLen(opt[seqPos]); + storeStart--; + DEBUGLOG(6, "sequence from rPos=%u copied into pos=%u (llen=%u,mlen=%u,ofc=%u)", + seqPos, storeStart, opt[seqPos].litlen, opt[seqPos].mlen, opt[seqPos].off); + opt[storeStart] = opt[seqPos]; + seqPos = (seqPos > backDist) ? seqPos - backDist : 0; + } + + /* save sequences */ + DEBUGLOG(6, "sending selected sequences into seqStore") + { U32 storePos; + for (storePos=storeStart; storePos <= storeEnd; storePos++) { + U32 const llen = opt[storePos].litlen; + U32 const mlen = opt[storePos].mlen; + U32 const offBase = opt[storePos].off; + U32 const advance = llen + mlen; + DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", + anchor - istart, (unsigned)llen, (unsigned)mlen); + + if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ + assert(storePos == storeEnd); /* must be last sequence */ + ip = anchor + llen; /* last "sequence" is a bunch of literals => don't progress anchor */ + continue; /* will finish */ + } + + assert(anchor + llen <= iend); + ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen); + anchor += advance; + ip = anchor; + } } + ZSTD_setBasePrices(optStatePtr, optLevel); + } + } /* while (ip < ilimit) */ + + /* Return the last literals size */ + return (size_t)(iend - anchor); +} + +static size_t ZSTD_compressBlock_opt0( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 0 /* optLevel */, dictMode); +} + +static size_t ZSTD_compressBlock_opt2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize, const ZSTD_dictMode_e dictMode) +{ + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /* optLevel */, dictMode); +} + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btopt"); + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + + + + +/* ZSTD_initStats_ultra(): + * make a first compression pass, just to seed stats with more accurate starting values. + * only works on first block, with no dictionary and no ldm. + * this function cannot error out, its narrow contract must be respected. + */ +static void +ZSTD_initStats_ultra(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + ZSTD_memcpy(tmpRep, rep, sizeof(tmpRep)); + + DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); + assert(ms->opt.litLengthSum == 0); /* first block */ + assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ + assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ + assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ + + ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/ + + /* invalidate first scan from history, only keep entropy stats */ + ZSTD_resetSeqStore(seqStore); + ms->window.base -= srcSize; + ms->window.dictLimit += (U32)srcSize; + ms->window.lowLimit = ms->window.dictLimit; + ms->nextToUpdate = ms->window.dictLimit; + +} + +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 const curr = (U32)((const BYTE*)src - ms->window.base); + DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + + /* 2-passes strategy: + * this strategy makes a first pass over first block to collect statistics + * in order to seed next round's statistics with it. + * After 1st pass, function forgets history, and starts a new block. + * Consequently, this can only work if no data has been previously loaded in tables, + * aka, no dictionary, no prefix, no ldm preprocessing. + * The compression ratio gain is generally small (~0.5% on first block), + ** the cost is 2x cpu time on first block. */ + assert(srcSize <= ZSTD_BLOCKSIZE_MAX); + if ( (ms->opt.litLengthSum==0) /* first block */ + && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ + && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */ + ) { + ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); + } + + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_dictMatchState); +} + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt0(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} + +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_opt2(ms, seqStore, rep, src, srcSize, ZSTD_extDict); +} + +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.h b/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.h new file mode 100644 index 000000000..342e5a311 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstd_opt.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_OPT_H +#define ZSTD_OPT_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "zstd_compress_internal.h" + +/* used in ZSTD_loadDictionaryContent() */ +void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend); + +size_t ZSTD_compressBlock_btopt( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + +size_t ZSTD_compressBlock_btopt_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_dictMatchState( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + +size_t ZSTD_compressBlock_btopt_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra_extDict( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + + /* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_OPT_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.c b/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.c new file mode 100644 index 000000000..678607556 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.c @@ -0,0 +1,1867 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +/* ====== Constants ====== */ +#define ZSTDMT_OVERLAPLOG_DEFAULT 0 + + +/* ====== Dependencies ====== */ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */ +#include "../common/mem.h" /* MEM_STATIC */ +#include "../common/pool.h" /* threadpool */ +#include "../common/threading.h" /* mutex */ +#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ +#include "zstd_ldm.h" +#include "zstdmt_compress.h" + +/* Guards code to support resizing the SeqPool. + * We will want to resize the SeqPool to save memory in the future. + * Until then, comment the code out since it is unused. + */ +#define ZSTD_RESIZE_SEQPOOL 0 + +/* ====== Debug ====== */ +#if defined(DEBUGLEVEL) && (DEBUGLEVEL>=2) \ + && !defined(_MSC_VER) \ + && !defined(__MINGW32__) + +# include +# include +# include + +# define DEBUG_PRINTHEX(l,p,n) { \ + unsigned debug_u; \ + for (debug_u=0; debug_u<(n); debug_u++) \ + RAWLOG(l, "%02X ", ((const unsigned char*)(p))[debug_u]); \ + RAWLOG(l, " \n"); \ +} + +static unsigned long long GetCurrentClockTimeMicroseconds(void) +{ + static clock_t _ticksPerSecond = 0; + if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); + + { struct tms junk; clock_t newTicks = (clock_t) times(&junk); + return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); +} } + +#define MUTEX_WAIT_TIME_DLEVEL 6 +#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \ + if (DEBUGLEVEL >= MUTEX_WAIT_TIME_DLEVEL) { \ + unsigned long long const beforeTime = GetCurrentClockTimeMicroseconds(); \ + ZSTD_pthread_mutex_lock(mutex); \ + { unsigned long long const afterTime = GetCurrentClockTimeMicroseconds(); \ + unsigned long long const elapsedTime = (afterTime-beforeTime); \ + if (elapsedTime > 1000) { /* or whatever threshold you like; I'm using 1 millisecond here */ \ + DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ + elapsedTime, #mutex); \ + } } \ + } else { \ + ZSTD_pthread_mutex_lock(mutex); \ + } \ +} + +#else + +# define ZSTD_PTHREAD_MUTEX_LOCK(m) ZSTD_pthread_mutex_lock(m) +# define DEBUG_PRINTHEX(l,p,n) {} + +#endif + + +/* ===== Buffer Pool ===== */ +/* a single Buffer Pool can be invoked from multiple threads in parallel */ + +typedef struct buffer_s { + void* start; + size_t capacity; +} buffer_t; + +static const buffer_t g_nullBuffer = { NULL, 0 }; + +typedef struct ZSTDMT_bufferPool_s { + ZSTD_pthread_mutex_t poolMutex; + size_t bufferSize; + unsigned totalBuffers; + unsigned nbBuffers; + ZSTD_customMem cMem; + buffer_t bTable[1]; /* variable size */ +} ZSTDMT_bufferPool; + +static ZSTDMT_bufferPool* ZSTDMT_createBufferPool(unsigned maxNbBuffers, ZSTD_customMem cMem) +{ + ZSTDMT_bufferPool* const bufPool = (ZSTDMT_bufferPool*)ZSTD_customCalloc( + sizeof(ZSTDMT_bufferPool) + (maxNbBuffers-1) * sizeof(buffer_t), cMem); + if (bufPool==NULL) return NULL; + if (ZSTD_pthread_mutex_init(&bufPool->poolMutex, NULL)) { + ZSTD_customFree(bufPool, cMem); + return NULL; + } + bufPool->bufferSize = 64 KB; + bufPool->totalBuffers = maxNbBuffers; + bufPool->nbBuffers = 0; + bufPool->cMem = cMem; + return bufPool; +} + +static void ZSTDMT_freeBufferPool(ZSTDMT_bufferPool* bufPool) +{ + unsigned u; + DEBUGLOG(3, "ZSTDMT_freeBufferPool (address:%08X)", (U32)(size_t)bufPool); + if (!bufPool) return; /* compatibility with free on NULL */ + for (u=0; utotalBuffers; u++) { + DEBUGLOG(4, "free buffer %2u (address:%08X)", u, (U32)(size_t)bufPool->bTable[u].start); + ZSTD_customFree(bufPool->bTable[u].start, bufPool->cMem); + } + ZSTD_pthread_mutex_destroy(&bufPool->poolMutex); + ZSTD_customFree(bufPool, bufPool->cMem); +} + +/* only works at initialization, not during compression */ +static size_t ZSTDMT_sizeof_bufferPool(ZSTDMT_bufferPool* bufPool) +{ + size_t const poolSize = sizeof(*bufPool) + + (bufPool->totalBuffers - 1) * sizeof(buffer_t); + unsigned u; + size_t totalBufferSize = 0; + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + for (u=0; utotalBuffers; u++) + totalBufferSize += bufPool->bTable[u].capacity; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + + return poolSize + totalBufferSize; +} + +/* ZSTDMT_setBufferSize() : + * all future buffers provided by this buffer pool will have _at least_ this size + * note : it's better for all buffers to have same size, + * as they become freely interchangeable, reducing malloc/free usages and memory fragmentation */ +static void ZSTDMT_setBufferSize(ZSTDMT_bufferPool* const bufPool, size_t const bSize) +{ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + DEBUGLOG(4, "ZSTDMT_setBufferSize: bSize = %u", (U32)bSize); + bufPool->bufferSize = bSize; + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); +} + + +static ZSTDMT_bufferPool* ZSTDMT_expandBufferPool(ZSTDMT_bufferPool* srcBufPool, unsigned maxNbBuffers) +{ + if (srcBufPool==NULL) return NULL; + if (srcBufPool->totalBuffers >= maxNbBuffers) /* good enough */ + return srcBufPool; + /* need a larger buffer pool */ + { ZSTD_customMem const cMem = srcBufPool->cMem; + size_t const bSize = srcBufPool->bufferSize; /* forward parameters */ + ZSTDMT_bufferPool* newBufPool; + ZSTDMT_freeBufferPool(srcBufPool); + newBufPool = ZSTDMT_createBufferPool(maxNbBuffers, cMem); + if (newBufPool==NULL) return newBufPool; + ZSTDMT_setBufferSize(newBufPool, bSize); + return newBufPool; + } +} + +/** ZSTDMT_getBuffer() : + * assumption : bufPool must be valid + * @return : a buffer, with start pointer and size + * note: allocation may fail, in this case, start==NULL and size==0 */ +static buffer_t ZSTDMT_getBuffer(ZSTDMT_bufferPool* bufPool) +{ + size_t const bSize = bufPool->bufferSize; + DEBUGLOG(5, "ZSTDMT_getBuffer: bSize = %u", (U32)bufPool->bufferSize); + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers) { /* try to use an existing buffer */ + buffer_t const buf = bufPool->bTable[--(bufPool->nbBuffers)]; + size_t const availBufferSize = buf.capacity; + bufPool->bTable[bufPool->nbBuffers] = g_nullBuffer; + if ((availBufferSize >= bSize) & ((availBufferSize>>3) <= bSize)) { + /* large enough, but not too much */ + DEBUGLOG(5, "ZSTDMT_getBuffer: provide buffer %u of size %u", + bufPool->nbBuffers, (U32)buf.capacity); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return buf; + } + /* size conditions not respected : scratch this buffer, create new one */ + DEBUGLOG(5, "ZSTDMT_getBuffer: existing buffer does not meet size conditions => freeing"); + ZSTD_customFree(buf.start, bufPool->cMem); + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* create new buffer */ + DEBUGLOG(5, "ZSTDMT_getBuffer: create a new buffer"); + { buffer_t buffer; + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer.start = start; /* note : start can be NULL if malloc fails ! */ + buffer.capacity = (start==NULL) ? 0 : bSize; + if (start==NULL) { + DEBUGLOG(5, "ZSTDMT_getBuffer: buffer allocation failure !!"); + } else { + DEBUGLOG(5, "ZSTDMT_getBuffer: created buffer of size %u", (U32)bSize); + } + return buffer; + } +} + +#if ZSTD_RESIZE_SEQPOOL +/** ZSTDMT_resizeBuffer() : + * assumption : bufPool must be valid + * @return : a buffer that is at least the buffer pool buffer size. + * If a reallocation happens, the data in the input buffer is copied. + */ +static buffer_t ZSTDMT_resizeBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buffer) +{ + size_t const bSize = bufPool->bufferSize; + if (buffer.capacity < bSize) { + void* const start = ZSTD_customMalloc(bSize, bufPool->cMem); + buffer_t newBuffer; + newBuffer.start = start; + newBuffer.capacity = start == NULL ? 0 : bSize; + if (start != NULL) { + assert(newBuffer.capacity >= buffer.capacity); + ZSTD_memcpy(newBuffer.start, buffer.start, buffer.capacity); + DEBUGLOG(5, "ZSTDMT_resizeBuffer: created buffer of size %u", (U32)bSize); + return newBuffer; + } + DEBUGLOG(5, "ZSTDMT_resizeBuffer: buffer allocation failure !!"); + } + return buffer; +} +#endif + +/* store buffer for later re-use, up to pool capacity */ +static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) +{ + DEBUGLOG(5, "ZSTDMT_releaseBuffer"); + if (buf.start == NULL) return; /* compatible with release on NULL */ + ZSTD_pthread_mutex_lock(&bufPool->poolMutex); + if (bufPool->nbBuffers < bufPool->totalBuffers) { + bufPool->bTable[bufPool->nbBuffers++] = buf; /* stored for later use */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: stored buffer of size %u in slot %u", + (U32)buf.capacity, (U32)(bufPool->nbBuffers-1)); + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + return; + } + ZSTD_pthread_mutex_unlock(&bufPool->poolMutex); + /* Reached bufferPool capacity (should not happen) */ + DEBUGLOG(5, "ZSTDMT_releaseBuffer: pool capacity reached => freeing "); + ZSTD_customFree(buf.start, bufPool->cMem); +} + +/* We need 2 output buffers per worker since each dstBuff must be flushed after it is released. + * The 3 additional buffers are as follows: + * 1 buffer for input loading + * 1 buffer for "next input" when submitting current one + * 1 buffer stuck in queue */ +#define BUF_POOL_MAX_NB_BUFFERS(nbWorkers) (2*(nbWorkers) + 3) + +/* After a worker releases its rawSeqStore, it is immediately ready for reuse. + * So we only need one seq buffer per worker. */ +#define SEQ_POOL_MAX_NB_BUFFERS(nbWorkers) (nbWorkers) + +/* ===== Seq Pool Wrapper ====== */ + +typedef ZSTDMT_bufferPool ZSTDMT_seqPool; + +static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool) +{ + return ZSTDMT_sizeof_bufferPool(seqPool); +} + +static rawSeqStore_t bufferToSeq(buffer_t buffer) +{ + rawSeqStore_t seq = kNullRawSeqStore; + seq.seq = (rawSeq*)buffer.start; + seq.capacity = buffer.capacity / sizeof(rawSeq); + return seq; +} + +static buffer_t seqToBuffer(rawSeqStore_t seq) +{ + buffer_t buffer; + buffer.start = seq.seq; + buffer.capacity = seq.capacity * sizeof(rawSeq); + return buffer; +} + +static rawSeqStore_t ZSTDMT_getSeq(ZSTDMT_seqPool* seqPool) +{ + if (seqPool->bufferSize == 0) { + return kNullRawSeqStore; + } + return bufferToSeq(ZSTDMT_getBuffer(seqPool)); +} + +#if ZSTD_RESIZE_SEQPOOL +static rawSeqStore_t ZSTDMT_resizeSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + return bufferToSeq(ZSTDMT_resizeBuffer(seqPool, seqToBuffer(seq))); +} +#endif + +static void ZSTDMT_releaseSeq(ZSTDMT_seqPool* seqPool, rawSeqStore_t seq) +{ + ZSTDMT_releaseBuffer(seqPool, seqToBuffer(seq)); +} + +static void ZSTDMT_setNbSeq(ZSTDMT_seqPool* const seqPool, size_t const nbSeq) +{ + ZSTDMT_setBufferSize(seqPool, nbSeq * sizeof(rawSeq)); +} + +static ZSTDMT_seqPool* ZSTDMT_createSeqPool(unsigned nbWorkers, ZSTD_customMem cMem) +{ + ZSTDMT_seqPool* const seqPool = ZSTDMT_createBufferPool(SEQ_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + if (seqPool == NULL) return NULL; + ZSTDMT_setNbSeq(seqPool, 0); + return seqPool; +} + +static void ZSTDMT_freeSeqPool(ZSTDMT_seqPool* seqPool) +{ + ZSTDMT_freeBufferPool(seqPool); +} + +static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) +{ + return ZSTDMT_expandBufferPool(pool, SEQ_POOL_MAX_NB_BUFFERS(nbWorkers)); +} + + +/* ===== CCtx Pool ===== */ +/* a single CCtx Pool can be invoked from multiple threads in parallel */ + +typedef struct { + ZSTD_pthread_mutex_t poolMutex; + int totalCCtx; + int availCCtx; + ZSTD_customMem cMem; + ZSTD_CCtx* cctx[1]; /* variable size */ +} ZSTDMT_CCtxPool; + +/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ +static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) +{ + int cid; + for (cid=0; cidtotalCCtx; cid++) + ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ + ZSTD_pthread_mutex_destroy(&pool->poolMutex); + ZSTD_customFree(pool, pool->cMem); +} + +/* ZSTDMT_createCCtxPool() : + * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, + ZSTD_customMem cMem) +{ + ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_customCalloc( + sizeof(ZSTDMT_CCtxPool) + (nbWorkers-1)*sizeof(ZSTD_CCtx*), cMem); + assert(nbWorkers > 0); + if (!cctxPool) return NULL; + if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { + ZSTD_customFree(cctxPool, cMem); + return NULL; + } + cctxPool->cMem = cMem; + cctxPool->totalCCtx = nbWorkers; + cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ + cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } + DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); + return cctxPool; +} + +static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, + int nbWorkers) +{ + if (srcPool==NULL) return NULL; + if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ + /* need a larger cctx pool */ + { ZSTD_customMem const cMem = srcPool->cMem; + ZSTDMT_freeCCtxPool(srcPool); + return ZSTDMT_createCCtxPool(nbWorkers, cMem); + } +} + +/* only works during initialization phase, not during compression */ +static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) +{ + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + { unsigned const nbWorkers = cctxPool->totalCCtx; + size_t const poolSize = sizeof(*cctxPool) + + (nbWorkers-1) * sizeof(ZSTD_CCtx*); + unsigned u; + size_t totalCCtxSize = 0; + for (u=0; ucctx[u]); + } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + assert(nbWorkers > 0); + return poolSize + totalCCtxSize; + } +} + +static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) +{ + DEBUGLOG(5, "ZSTDMT_getCCtx"); + ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); + if (cctxPool->availCCtx) { + cctxPool->availCCtx--; + { ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + return cctx; + } } + ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); + DEBUGLOG(5, "create one more CCtx"); + return ZSTD_createCCtx_advanced(cctxPool->cMem); /* note : can be NULL, when creation fails ! */ +} + +static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return; /* compatibility with release on NULL */ + ZSTD_pthread_mutex_lock(&pool->poolMutex); + if (pool->availCCtx < pool->totalCCtx) + pool->cctx[pool->availCCtx++] = cctx; + else { + /* pool overflow : should not happen, since totalCCtx==nbWorkers */ + DEBUGLOG(4, "CCtx pool overflow : free cctx"); + ZSTD_freeCCtx(cctx); + } + ZSTD_pthread_mutex_unlock(&pool->poolMutex); +} + +/* ==== Serial State ==== */ + +typedef struct { + void const* start; + size_t size; +} range_t; + +typedef struct { + /* All variables in the struct are protected by mutex. */ + ZSTD_pthread_mutex_t mutex; + ZSTD_pthread_cond_t cond; + ZSTD_CCtx_params params; + ldmState_t ldmState; + XXH64_state_t xxhState; + unsigned nextJobID; + /* Protects ldmWindow. + * Must be acquired after the main mutex when acquiring both. + */ + ZSTD_pthread_mutex_t ldmWindowMutex; + ZSTD_pthread_cond_t ldmWindowCond; /* Signaled when ldmWindow is updated */ + ZSTD_window_t ldmWindow; /* A thread-safe copy of ldmState.window */ +} serialState_t; + +static int +ZSTDMT_serialState_reset(serialState_t* serialState, + ZSTDMT_seqPool* seqPool, + ZSTD_CCtx_params params, + size_t jobSize, + const void* dict, size_t const dictSize, + ZSTD_dictContentType_e dictContentType) +{ + /* Adjust parameters */ + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); + ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); + assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); + assert(params.ldmParams.hashRateLog < 32); + } else { + ZSTD_memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); + } + serialState->nextJobID = 0; + if (params.fParams.checksumFlag) + XXH64_reset(&serialState->xxhState, 0); + if (params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_customMem cMem = params.customMem; + unsigned const hashLog = params.ldmParams.hashLog; + size_t const hashSize = ((size_t)1 << hashLog) * sizeof(ldmEntry_t); + unsigned const bucketLog = + params.ldmParams.hashLog - params.ldmParams.bucketSizeLog; + unsigned const prevBucketLog = + serialState->params.ldmParams.hashLog - + serialState->params.ldmParams.bucketSizeLog; + size_t const numBuckets = (size_t)1 << bucketLog; + /* Size the seq pool tables */ + ZSTDMT_setNbSeq(seqPool, ZSTD_ldm_getMaxNbSeq(params.ldmParams, jobSize)); + /* Reset the window */ + ZSTD_window_init(&serialState->ldmState.window); + /* Resize tables and output space if necessary. */ + if (serialState->ldmState.hashTable == NULL || serialState->params.ldmParams.hashLog < hashLog) { + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + serialState->ldmState.hashTable = (ldmEntry_t*)ZSTD_customMalloc(hashSize, cMem); + } + if (serialState->ldmState.bucketOffsets == NULL || prevBucketLog < bucketLog) { + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); + serialState->ldmState.bucketOffsets = (BYTE*)ZSTD_customMalloc(numBuckets, cMem); + } + if (!serialState->ldmState.hashTable || !serialState->ldmState.bucketOffsets) + return 1; + /* Zero the tables */ + ZSTD_memset(serialState->ldmState.hashTable, 0, hashSize); + ZSTD_memset(serialState->ldmState.bucketOffsets, 0, numBuckets); + + /* Update window state and fill hash table with dict */ + serialState->ldmState.loadedDictEnd = 0; + if (dictSize > 0) { + if (dictContentType == ZSTD_dct_rawContent) { + BYTE const* const dictEnd = (const BYTE*)dict + dictSize; + ZSTD_window_update(&serialState->ldmState.window, dict, dictSize, /* forceNonContiguous */ 0); + ZSTD_ldm_fillHashTable(&serialState->ldmState, (const BYTE*)dict, dictEnd, ¶ms.ldmParams); + serialState->ldmState.loadedDictEnd = params.forceWindow ? 0 : (U32)(dictEnd - serialState->ldmState.window.base); + } else { + /* don't even load anything */ + } + } + + /* Initialize serialState's copy of ldmWindow. */ + serialState->ldmWindow = serialState->ldmState.window; + } + + serialState->params = params; + serialState->params.jobSize = (U32)jobSize; + return 0; +} + +static int ZSTDMT_serialState_init(serialState_t* serialState) +{ + int initError = 0; + ZSTD_memset(serialState, 0, sizeof(*serialState)); + initError |= ZSTD_pthread_mutex_init(&serialState->mutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->cond, NULL); + initError |= ZSTD_pthread_mutex_init(&serialState->ldmWindowMutex, NULL); + initError |= ZSTD_pthread_cond_init(&serialState->ldmWindowCond, NULL); + return initError; +} + +static void ZSTDMT_serialState_free(serialState_t* serialState) +{ + ZSTD_customMem cMem = serialState->params.customMem; + ZSTD_pthread_mutex_destroy(&serialState->mutex); + ZSTD_pthread_cond_destroy(&serialState->cond); + ZSTD_pthread_mutex_destroy(&serialState->ldmWindowMutex); + ZSTD_pthread_cond_destroy(&serialState->ldmWindowCond); + ZSTD_customFree(serialState->ldmState.hashTable, cMem); + ZSTD_customFree(serialState->ldmState.bucketOffsets, cMem); +} + +static void ZSTDMT_serialState_update(serialState_t* serialState, + ZSTD_CCtx* jobCCtx, rawSeqStore_t seqStore, + range_t src, unsigned jobID) +{ + /* Wait for our turn */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + while (serialState->nextJobID < jobID) { + DEBUGLOG(5, "wait for serialState->cond"); + ZSTD_pthread_cond_wait(&serialState->cond, &serialState->mutex); + } + /* A future job may error and skip our job */ + if (serialState->nextJobID == jobID) { + /* It is now our turn, do any processing necessary */ + if (serialState->params.ldmParams.enableLdm == ZSTD_ps_enable) { + size_t error; + assert(seqStore.seq != NULL && seqStore.pos == 0 && + seqStore.size == 0 && seqStore.capacity > 0); + assert(src.size <= serialState->params.jobSize); + ZSTD_window_update(&serialState->ldmState.window, src.start, src.size, /* forceNonContiguous */ 0); + error = ZSTD_ldm_generateSequences( + &serialState->ldmState, &seqStore, + &serialState->params.ldmParams, src.start, src.size); + /* We provide a large enough buffer to never fail. */ + assert(!ZSTD_isError(error)); (void)error; + /* Update ldmWindow to match the ldmState.window and signal the main + * thread if it is waiting for a buffer. + */ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + serialState->ldmWindow = serialState->ldmState.window; + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + if (serialState->params.fParams.checksumFlag && src.size > 0) + XXH64_update(&serialState->xxhState, src.start, src.size); + } + /* Now it is the next jobs turn */ + serialState->nextJobID++; + ZSTD_pthread_cond_broadcast(&serialState->cond); + ZSTD_pthread_mutex_unlock(&serialState->mutex); + + if (seqStore.size > 0) { + size_t const err = ZSTD_referenceExternalSequences( + jobCCtx, seqStore.seq, seqStore.size); + assert(serialState->params.ldmParams.enableLdm == ZSTD_ps_enable); + assert(!ZSTD_isError(err)); + (void)err; + } +} + +static void ZSTDMT_serialState_ensureFinished(serialState_t* serialState, + unsigned jobID, size_t cSize) +{ + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->mutex); + if (serialState->nextJobID <= jobID) { + assert(ZSTD_isError(cSize)); (void)cSize; + DEBUGLOG(5, "Skipping past job %u because of error", jobID); + serialState->nextJobID = jobID + 1; + ZSTD_pthread_cond_broadcast(&serialState->cond); + + ZSTD_PTHREAD_MUTEX_LOCK(&serialState->ldmWindowMutex); + ZSTD_window_clear(&serialState->ldmWindow); + ZSTD_pthread_cond_signal(&serialState->ldmWindowCond); + ZSTD_pthread_mutex_unlock(&serialState->ldmWindowMutex); + } + ZSTD_pthread_mutex_unlock(&serialState->mutex); + +} + + +/* ------------------------------------------ */ +/* ===== Worker thread ===== */ +/* ------------------------------------------ */ + +static const range_t kNullRange = { NULL, 0 }; + +typedef struct { + size_t consumed; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx */ + size_t cSize; /* SHARED - set0 by mtctx, then modified by worker AND read by mtctx, then set0 by mtctx */ + ZSTD_pthread_mutex_t job_mutex; /* Thread-safe - used by mtctx and worker */ + ZSTD_pthread_cond_t job_cond; /* Thread-safe - used by mtctx and worker */ + ZSTDMT_CCtxPool* cctxPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_bufferPool* bufPool; /* Thread-safe - used by mtctx and (all) workers */ + ZSTDMT_seqPool* seqPool; /* Thread-safe - used by mtctx and (all) workers */ + serialState_t* serial; /* Thread-safe - used by mtctx and (all) workers */ + buffer_t dstBuff; /* set by worker (or mtctx), then read by worker & mtctx, then modified by mtctx => no barrier */ + range_t prefix; /* set by mtctx, then read by worker & mtctx => no barrier */ + range_t src; /* set by mtctx, then read by worker & mtctx => no barrier */ + unsigned jobID; /* set by mtctx, then read by worker => no barrier */ + unsigned firstJob; /* set by mtctx, then read by worker => no barrier */ + unsigned lastJob; /* set by mtctx, then read by worker => no barrier */ + ZSTD_CCtx_params params; /* set by mtctx, then read by worker => no barrier */ + const ZSTD_CDict* cdict; /* set by mtctx, then read by worker => no barrier */ + unsigned long long fullFrameSize; /* set by mtctx, then read by worker => no barrier */ + size_t dstFlushed; /* used only by mtctx */ + unsigned frameChecksumNeeded; /* used only by mtctx */ +} ZSTDMT_jobDescription; + +#define JOB_ERROR(e) { \ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); \ + job->cSize = e; \ + ZSTD_pthread_mutex_unlock(&job->job_mutex); \ + goto _endJob; \ +} + +/* ZSTDMT_compressionJob() is a POOL_function type */ +static void ZSTDMT_compressionJob(void* jobDescription) +{ + ZSTDMT_jobDescription* const job = (ZSTDMT_jobDescription*)jobDescription; + ZSTD_CCtx_params jobParams = job->params; /* do not modify job->params ! copy it, modify the copy */ + ZSTD_CCtx* const cctx = ZSTDMT_getCCtx(job->cctxPool); + rawSeqStore_t rawSeqStore = ZSTDMT_getSeq(job->seqPool); + buffer_t dstBuff = job->dstBuff; + size_t lastCBlockSize = 0; + + /* resources */ + if (cctx==NULL) JOB_ERROR(ERROR(memory_allocation)); + if (dstBuff.start == NULL) { /* streaming job : doesn't provide a dstBuffer */ + dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (dstBuff.start==NULL) JOB_ERROR(ERROR(memory_allocation)); + job->dstBuff = dstBuff; /* this value can be read in ZSTDMT_flush, when it copies the whole job */ + } + if (jobParams.ldmParams.enableLdm == ZSTD_ps_enable && rawSeqStore.seq == NULL) + JOB_ERROR(ERROR(memory_allocation)); + + /* Don't compute the checksum for chunks, since we compute it externally, + * but write it in the header. + */ + if (job->jobID != 0) jobParams.fParams.checksumFlag = 0; + /* Don't run LDM for the chunks, since we handle it externally */ + jobParams.ldmParams.enableLdm = ZSTD_ps_disable; + /* Correct nbWorkers to 0. */ + jobParams.nbWorkers = 0; + + + /* init */ + if (job->cdict) { + size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, NULL, 0, ZSTD_dct_auto, ZSTD_dtlm_fast, job->cdict, &jobParams, job->fullFrameSize); + assert(job->firstJob); /* only allowed for first job */ + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } else { /* srcStart points at reloaded section */ + U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; + { size_t const forceWindowError = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); + if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); + } + if (!job->firstJob) { + size_t const err = ZSTD_CCtxParams_setParameter(&jobParams, ZSTD_c_deterministicRefPrefix, 0); + if (ZSTD_isError(err)) JOB_ERROR(err); + } + { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, + job->prefix.start, job->prefix.size, ZSTD_dct_rawContent, /* load dictionary in "content-only" mode (no header analysis) */ + ZSTD_dtlm_fast, + NULL, /*cdict*/ + &jobParams, pledgedSrcSize); + if (ZSTD_isError(initError)) JOB_ERROR(initError); + } } + + /* Perform serial step as early as possible, but after CCtx initialization */ + ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); + + if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ + size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); + if (ZSTD_isError(hSize)) JOB_ERROR(hSize); + DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); + ZSTD_invalidateRepCodes(cctx); + } + + /* compress */ + { size_t const chunkSize = 4*ZSTD_BLOCKSIZE_MAX; + int const nbChunks = (int)((job->src.size + (chunkSize-1)) / chunkSize); + const BYTE* ip = (const BYTE*) job->src.start; + BYTE* const ostart = (BYTE*)dstBuff.start; + BYTE* op = ostart; + BYTE* oend = op + dstBuff.capacity; + int chunkNb; + if (sizeof(size_t) > sizeof(int)) assert(job->src.size < ((size_t)INT_MAX) * chunkSize); /* check overflow */ + DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); + assert(job->cSize == 0); + for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { + size_t const cSize = ZSTD_compressContinue_public(cctx, op, oend-op, ip, chunkSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + ip += chunkSize; + op += cSize; assert(op < oend); + /* stats */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + job->cSize += cSize; + job->consumed = chunkSize * chunkNb; + DEBUGLOG(5, "ZSTDMT_compressionJob: compress new block : cSize==%u bytes (total: %u)", + (U32)cSize, (U32)job->cSize); + ZSTD_pthread_cond_signal(&job->job_cond); /* warns some more data is ready to be flushed */ + ZSTD_pthread_mutex_unlock(&job->job_mutex); + } + /* last block */ + assert(chunkSize > 0); + assert((chunkSize & (chunkSize - 1)) == 0); /* chunkSize must be power of 2 for mask==(chunkSize-1) to work */ + if ((nbChunks > 0) | job->lastJob /*must output a "last block" flag*/ ) { + size_t const lastBlockSize1 = job->src.size & (chunkSize-1); + size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1; + size_t const cSize = (job->lastJob) ? + ZSTD_compressEnd_public(cctx, op, oend-op, ip, lastBlockSize) : + ZSTD_compressContinue_public(cctx, op, oend-op, ip, lastBlockSize); + if (ZSTD_isError(cSize)) JOB_ERROR(cSize); + lastCBlockSize = cSize; + } } + if (!job->firstJob) { + /* Double check that we don't have an ext-dict, because then our + * repcode invalidation doesn't work. + */ + assert(!ZSTD_window_hasExtDict(cctx->blockState.matchState.window)); + } + ZSTD_CCtx_trace(cctx, 0); + +_endJob: + ZSTDMT_serialState_ensureFinished(job->serial, job->jobID, job->cSize); + if (job->prefix.size > 0) + DEBUGLOG(5, "Finished with prefix: %zx", (size_t)job->prefix.start); + DEBUGLOG(5, "Finished with source: %zx", (size_t)job->src.start); + /* release resources */ + ZSTDMT_releaseSeq(job->seqPool, rawSeqStore); + ZSTDMT_releaseCCtx(job->cctxPool, cctx); + /* report */ + ZSTD_PTHREAD_MUTEX_LOCK(&job->job_mutex); + if (ZSTD_isError(job->cSize)) assert(lastCBlockSize == 0); + job->cSize += lastCBlockSize; + job->consumed = job->src.size; /* when job->consumed == job->src.size , compression job is presumed completed */ + ZSTD_pthread_cond_signal(&job->job_cond); + ZSTD_pthread_mutex_unlock(&job->job_mutex); +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +typedef struct { + range_t prefix; /* read-only non-owned prefix buffer */ + buffer_t buffer; + size_t filled; +} inBuff_t; + +typedef struct { + BYTE* buffer; /* The round input buffer. All jobs get references + * to pieces of the buffer. ZSTDMT_tryGetInputRange() + * handles handing out job input buffers, and makes + * sure it doesn't overlap with any pieces still in use. + */ + size_t capacity; /* The capacity of buffer. */ + size_t pos; /* The position of the current inBuff in the round + * buffer. Updated past the end if the inBuff once + * the inBuff is sent to the worker thread. + * pos <= capacity. + */ +} roundBuff_t; + +static const roundBuff_t kNullRoundBuff = {NULL, 0, 0}; + +#define RSYNC_LENGTH 32 +/* Don't create chunks smaller than the zstd block size. + * This stops us from regressing compression ratio too much, + * and ensures our output fits in ZSTD_compressBound(). + * + * If this is shrunk < ZSTD_BLOCKSIZELOG_MIN then + * ZSTD_COMPRESSBOUND() will need to be updated. + */ +#define RSYNC_MIN_BLOCK_LOG ZSTD_BLOCKSIZELOG_MAX +#define RSYNC_MIN_BLOCK_SIZE (1< one job is already prepared, but pool has shortage of workers. Don't create a new job. */ + inBuff_t inBuff; + roundBuff_t roundBuff; + serialState_t serial; + rsyncState_t rsync; + unsigned jobIDMask; + unsigned doneJobID; + unsigned nextJobID; + unsigned frameEnded; + unsigned allJobsCompleted; + unsigned long long frameContentSize; + unsigned long long consumed; + unsigned long long produced; + ZSTD_customMem cMem; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + unsigned providedFactory: 1; +}; + +static void ZSTDMT_freeJobsTable(ZSTDMT_jobDescription* jobTable, U32 nbJobs, ZSTD_customMem cMem) +{ + U32 jobNb; + if (jobTable == NULL) return; + for (jobNb=0; jobNb mtctx->jobIDMask+1) { /* need more job capacity */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + mtctx->jobIDMask = 0; + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, mtctx->cMem); + if (mtctx->jobs==NULL) return ERROR(memory_allocation); + assert((nbJobs != 0) && ((nbJobs & (nbJobs - 1)) == 0)); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + } + return 0; +} + + +/* ZSTDMT_CCtxParam_setNbWorkers(): + * Internal use only */ +static size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorkers) +{ + return ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, (int)nbWorkers); +} + +MEM_STATIC ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced_internal(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ + ZSTDMT_CCtx* mtctx; + U32 nbJobs = nbWorkers + 2; + int initError; + DEBUGLOG(3, "ZSTDMT_createCCtx_advanced (nbWorkers = %u)", nbWorkers); + + if (nbWorkers < 1) return NULL; + nbWorkers = MIN(nbWorkers , ZSTDMT_NBWORKERS_MAX); + if ((cMem.customAlloc!=NULL) ^ (cMem.customFree!=NULL)) + /* invalid custom allocator */ + return NULL; + + mtctx = (ZSTDMT_CCtx*) ZSTD_customCalloc(sizeof(ZSTDMT_CCtx), cMem); + if (!mtctx) return NULL; + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + mtctx->cMem = cMem; + mtctx->allJobsCompleted = 1; + if (pool != NULL) { + mtctx->factory = pool; + mtctx->providedFactory = 1; + } + else { + mtctx->factory = POOL_create_advanced(nbWorkers, 0, cMem); + mtctx->providedFactory = 0; + } + mtctx->jobs = ZSTDMT_createJobsTable(&nbJobs, cMem); + assert(nbJobs > 0); assert((nbJobs & (nbJobs - 1)) == 0); /* ensure nbJobs is a power of 2 */ + mtctx->jobIDMask = nbJobs - 1; + mtctx->bufPool = ZSTDMT_createBufferPool(BUF_POOL_MAX_NB_BUFFERS(nbWorkers), cMem); + mtctx->cctxPool = ZSTDMT_createCCtxPool(nbWorkers, cMem); + mtctx->seqPool = ZSTDMT_createSeqPool(nbWorkers, cMem); + initError = ZSTDMT_serialState_init(&mtctx->serial); + mtctx->roundBuff = kNullRoundBuff; + if (!mtctx->factory | !mtctx->jobs | !mtctx->bufPool | !mtctx->cctxPool | !mtctx->seqPool | initError) { + ZSTDMT_freeCCtx(mtctx); + return NULL; + } + DEBUGLOG(3, "mt_cctx created, for %u threads", nbWorkers); + return mtctx; +} + +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, ZSTD_customMem cMem, ZSTD_threadPool* pool) +{ +#ifdef ZSTD_MULTITHREAD + return ZSTDMT_createCCtx_advanced_internal(nbWorkers, cMem, pool); +#else + (void)nbWorkers; + (void)cMem; + (void)pool; + return NULL; +#endif +} + + +/* ZSTDMT_releaseAllJobResources() : + * note : ensure all workers are killed first ! */ +static void ZSTDMT_releaseAllJobResources(ZSTDMT_CCtx* mtctx) +{ + unsigned jobID; + DEBUGLOG(3, "ZSTDMT_releaseAllJobResources"); + for (jobID=0; jobID <= mtctx->jobIDMask; jobID++) { + /* Copy the mutex/cond out */ + ZSTD_pthread_mutex_t const mutex = mtctx->jobs[jobID].job_mutex; + ZSTD_pthread_cond_t const cond = mtctx->jobs[jobID].job_cond; + + DEBUGLOG(4, "job%02u: release dst address %08X", jobID, (U32)(size_t)mtctx->jobs[jobID].dstBuff.start); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[jobID].dstBuff); + + /* Clear the job description, but keep the mutex/cond */ + ZSTD_memset(&mtctx->jobs[jobID], 0, sizeof(mtctx->jobs[jobID])); + mtctx->jobs[jobID].job_mutex = mutex; + mtctx->jobs[jobID].job_cond = cond; + } + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->allJobsCompleted = 1; +} + +static void ZSTDMT_waitForAllJobsCompleted(ZSTDMT_CCtx* mtctx) +{ + DEBUGLOG(4, "ZSTDMT_waitForAllJobsCompleted"); + while (mtctx->doneJobID < mtctx->nextJobID) { + unsigned const jobID = mtctx->doneJobID & mtctx->jobIDMask; + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[jobID].job_mutex); + while (mtctx->jobs[jobID].consumed < mtctx->jobs[jobID].src.size) { + DEBUGLOG(4, "waiting for jobCompleted signal from job %u", mtctx->doneJobID); /* we want to block when waiting for data to flush */ + ZSTD_pthread_cond_wait(&mtctx->jobs[jobID].job_cond, &mtctx->jobs[jobID].job_mutex); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[jobID].job_mutex); + mtctx->doneJobID++; + } +} + +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx==NULL) return 0; /* compatible with free on NULL */ + if (!mtctx->providedFactory) + POOL_free(mtctx->factory); /* stop and free worker threads */ + ZSTDMT_releaseAllJobResources(mtctx); /* release job resources into pools first */ + ZSTDMT_freeJobsTable(mtctx->jobs, mtctx->jobIDMask+1, mtctx->cMem); + ZSTDMT_freeBufferPool(mtctx->bufPool); + ZSTDMT_freeCCtxPool(mtctx->cctxPool); + ZSTDMT_freeSeqPool(mtctx->seqPool); + ZSTDMT_serialState_free(&mtctx->serial); + ZSTD_freeCDict(mtctx->cdictLocal); + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + ZSTD_customFree(mtctx, mtctx->cMem); + return 0; +} + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) +{ + if (mtctx == NULL) return 0; /* supports sizeof NULL */ + return sizeof(*mtctx) + + POOL_sizeof(mtctx->factory) + + ZSTDMT_sizeof_bufferPool(mtctx->bufPool) + + (mtctx->jobIDMask+1) * sizeof(ZSTDMT_jobDescription) + + ZSTDMT_sizeof_CCtxPool(mtctx->cctxPool) + + ZSTDMT_sizeof_seqPool(mtctx->seqPool) + + ZSTD_sizeof_CDict(mtctx->cdictLocal) + + mtctx->roundBuff.capacity; +} + + +/* ZSTDMT_resize() : + * @return : error code if fails, 0 on success */ +static size_t ZSTDMT_resize(ZSTDMT_CCtx* mtctx, unsigned nbWorkers) +{ + if (POOL_resize(mtctx->factory, nbWorkers)) return ERROR(memory_allocation); + FORWARD_IF_ERROR( ZSTDMT_expandJobsTable(mtctx, nbWorkers) , ""); + mtctx->bufPool = ZSTDMT_expandBufferPool(mtctx->bufPool, BUF_POOL_MAX_NB_BUFFERS(nbWorkers)); + if (mtctx->bufPool == NULL) return ERROR(memory_allocation); + mtctx->cctxPool = ZSTDMT_expandCCtxPool(mtctx->cctxPool, nbWorkers); + if (mtctx->cctxPool == NULL) return ERROR(memory_allocation); + mtctx->seqPool = ZSTDMT_expandSeqPool(mtctx->seqPool, nbWorkers); + if (mtctx->seqPool == NULL) return ERROR(memory_allocation); + ZSTDMT_CCtxParam_setNbWorkers(&mtctx->params, nbWorkers); + return 0; +} + + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates a selected set of compression parameters, remaining compatible with currently active frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams) +{ + U32 const saved_wlog = mtctx->params.cParams.windowLog; /* Do not modify windowLog while compressing */ + int const compressionLevel = cctxParams->compressionLevel; + DEBUGLOG(5, "ZSTDMT_updateCParams_whileCompressing (level:%i)", + compressionLevel); + mtctx->params.compressionLevel = compressionLevel; + { ZSTD_compressionParameters cParams = ZSTD_getCParamsFromCCtxParams(cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); + cParams.windowLog = saved_wlog; + mtctx->params.cParams = cParams; + } +} + +/* ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + * Note : mutex will be acquired during statistics collection inside workers. */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx) +{ + ZSTD_frameProgression fps; + DEBUGLOG(5, "ZSTDMT_getFrameProgression"); + fps.ingested = mtctx->consumed + mtctx->inBuff.filled; + fps.consumed = mtctx->consumed; + fps.produced = fps.flushed = mtctx->produced; + fps.currentJobID = mtctx->nextJobID; + fps.nbActiveWorkers = 0; + { unsigned jobNb; + unsigned lastJobNb = mtctx->nextJobID + mtctx->jobReady; assert(mtctx->jobReady <= 1); + DEBUGLOG(6, "ZSTDMT_getFrameProgression: jobs: from %u to <%u (jobReady:%u)", + mtctx->doneJobID, lastJobNb, mtctx->jobReady) + for (jobNb = mtctx->doneJobID ; jobNb < lastJobNb ; jobNb++) { + unsigned const wJobID = jobNb & mtctx->jobIDMask; + ZSTDMT_jobDescription* jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + fps.ingested += jobPtr->src.size; + fps.consumed += jobPtr->consumed; + fps.produced += produced; + fps.flushed += flushed; + fps.nbActiveWorkers += (jobPtr->consumed < jobPtr->src.size); + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + } + return fps; +} + + +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) +{ + size_t toFlush; + unsigned const jobID = mtctx->doneJobID; + assert(jobID <= mtctx->nextJobID); + if (jobID == mtctx->nextJobID) return 0; /* no active job => nothing to flush */ + + /* look into oldest non-fully-flushed job */ + { unsigned const wJobID = jobID & mtctx->jobIDMask; + ZSTDMT_jobDescription* const jobPtr = &mtctx->jobs[wJobID]; + ZSTD_pthread_mutex_lock(&jobPtr->job_mutex); + { size_t const cResult = jobPtr->cSize; + size_t const produced = ZSTD_isError(cResult) ? 0 : cResult; + size_t const flushed = ZSTD_isError(cResult) ? 0 : jobPtr->dstFlushed; + assert(flushed <= produced); + assert(jobPtr->consumed <= jobPtr->src.size); + toFlush = produced - flushed; + /* if toFlush==0, nothing is available to flush. + * However, jobID is expected to still be active: + * if jobID was already completed and fully flushed, + * ZSTDMT_flushProduced() should have already moved onto next job. + * Therefore, some input has not yet been consumed. */ + if (toFlush==0) { + assert(jobPtr->consumed < jobPtr->src.size); + } + } + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + } + + return toFlush; +} + + +/* ------------------------------------------ */ +/* ===== Multi-threaded compression ===== */ +/* ------------------------------------------ */ + +static unsigned ZSTDMT_computeTargetJobLog(const ZSTD_CCtx_params* params) +{ + unsigned jobLog; + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on cycleLog instead. */ + jobLog = MAX(21, ZSTD_cycleLog(params->cParams.chainLog, params->cParams.strategy) + 3); + } else { + jobLog = MAX(20, params->cParams.windowLog + 2); + } + return MIN(jobLog, (unsigned)ZSTDMT_JOBLOG_MAX); +} + +static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) +{ + switch(strat) + { + case ZSTD_btultra2: + return 9; + case ZSTD_btultra: + case ZSTD_btopt: + return 8; + case ZSTD_btlazy2: + case ZSTD_lazy2: + return 7; + case ZSTD_lazy: + case ZSTD_greedy: + case ZSTD_dfast: + case ZSTD_fast: + default:; + } + return 6; +} + +static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) +{ + assert(0 <= ovlog && ovlog <= 9); + if (ovlog == 0) return ZSTDMT_overlapLog_default(strat); + return ovlog; +} + +static size_t ZSTDMT_computeOverlapSize(const ZSTD_CCtx_params* params) +{ + int const overlapRLog = 9 - ZSTDMT_overlapLog(params->overlapLog, params->cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params->cParams.windowLog - overlapRLog); + assert(0 <= overlapRLog && overlapRLog <= 8); + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. + * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ + ovLog = MIN(params->cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + - overlapRLog; + } + assert(0 <= ovLog && ovLog <= ZSTD_WINDOWLOG_MAX); + DEBUGLOG(4, "overlapLog : %i", params->overlapLog); + DEBUGLOG(4, "overlap size : %i", 1 << ovLog); + return (ovLog==0) ? 0 : (size_t)1 << ovLog; +} + +/* ====================================== */ +/* ======= Streaming API ======= */ +/* ====================================== */ + +size_t ZSTDMT_initCStream_internal( + ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, ZSTD_CCtx_params params, + unsigned long long pledgedSrcSize) +{ + DEBUGLOG(4, "ZSTDMT_initCStream_internal (pledgedSrcSize=%u, nbWorkers=%u, cctxPool=%u)", + (U32)pledgedSrcSize, params.nbWorkers, mtctx->cctxPool->totalCCtx); + + /* params supposed partially fully validated at this point */ + assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); + assert(!((dict) && (cdict))); /* either dict or cdict, not both */ + + /* init */ + if (params.nbWorkers != mtctx->params.nbWorkers) + FORWARD_IF_ERROR( ZSTDMT_resize(mtctx, params.nbWorkers) , ""); + + if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = (size_t)ZSTDMT_JOBSIZE_MAX; + + DEBUGLOG(4, "ZSTDMT_initCStream_internal: %u workers", params.nbWorkers); + + if (mtctx->allJobsCompleted == 0) { /* previous compression not correctly finished */ + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + mtctx->allJobsCompleted = 1; + } + + mtctx->params = params; + mtctx->frameContentSize = pledgedSrcSize; + if (dict) { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, + ZSTD_dlm_byCopy, dictContentType, /* note : a loadPrefix becomes an internal CDict */ + params.cParams, mtctx->cMem); + mtctx->cdict = mtctx->cdictLocal; + if (mtctx->cdictLocal == NULL) return ERROR(memory_allocation); + } else { + ZSTD_freeCDict(mtctx->cdictLocal); + mtctx->cdictLocal = NULL; + mtctx->cdict = cdict; + } + + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(¶ms); + DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); + mtctx->targetSectionSize = params.jobSize; + if (mtctx->targetSectionSize == 0) { + mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(¶ms); + } + assert(mtctx->targetSectionSize <= (size_t)ZSTDMT_JOBSIZE_MAX); + + if (params.rsyncable) { + /* Aim for the targetsectionSize as the average job size. */ + U32 const jobSizeKB = (U32)(mtctx->targetSectionSize >> 10); + U32 const rsyncBits = (assert(jobSizeKB >= 1), ZSTD_highbit32(jobSizeKB) + 10); + /* We refuse to create jobs < RSYNC_MIN_BLOCK_SIZE bytes, so make sure our + * expected job size is at least 4x larger. */ + assert(rsyncBits >= RSYNC_MIN_BLOCK_LOG + 2); + DEBUGLOG(4, "rsyncLog = %u", rsyncBits); + mtctx->rsync.hash = 0; + mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; + mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); + } + if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ + DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize); + DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); + ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); + { + /* If ldm is enabled we need windowSize space. */ + size_t const windowSize = mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable ? (1U << mtctx->params.cParams.windowLog) : 0; + /* Two buffers of slack, plus extra space for the overlap + * This is the minimum slack that LDM works with. One extra because + * flush might waste up to targetSectionSize-1 bytes. Another extra + * for the overlap (if > 0), then one to fill which doesn't overlap + * with the LDM window. + */ + size_t const nbSlackBuffers = 2 + (mtctx->targetPrefixSize > 0); + size_t const slackSize = mtctx->targetSectionSize * nbSlackBuffers; + /* Compute the total size, and always have enough slack */ + size_t const nbWorkers = MAX(mtctx->params.nbWorkers, 1); + size_t const sectionsSize = mtctx->targetSectionSize * nbWorkers; + size_t const capacity = MAX(windowSize, sectionsSize) + slackSize; + if (mtctx->roundBuff.capacity < capacity) { + if (mtctx->roundBuff.buffer) + ZSTD_customFree(mtctx->roundBuff.buffer, mtctx->cMem); + mtctx->roundBuff.buffer = (BYTE*)ZSTD_customMalloc(capacity, mtctx->cMem); + if (mtctx->roundBuff.buffer == NULL) { + mtctx->roundBuff.capacity = 0; + return ERROR(memory_allocation); + } + mtctx->roundBuff.capacity = capacity; + } + } + DEBUGLOG(4, "roundBuff capacity : %u KB", (U32)(mtctx->roundBuff.capacity>>10)); + mtctx->roundBuff.pos = 0; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + mtctx->inBuff.prefix = kNullRange; + mtctx->doneJobID = 0; + mtctx->nextJobID = 0; + mtctx->frameEnded = 0; + mtctx->allJobsCompleted = 0; + mtctx->consumed = 0; + mtctx->produced = 0; + if (ZSTDMT_serialState_reset(&mtctx->serial, mtctx->seqPool, params, mtctx->targetSectionSize, + dict, dictSize, dictContentType)) + return ERROR(memory_allocation); + return 0; +} + + +/* ZSTDMT_writeLastEmptyBlock() + * Write a single empty block with an end-of-frame to finish a frame. + * Job must be created from streaming variant. + * This function is always successful if expected conditions are fulfilled. + */ +static void ZSTDMT_writeLastEmptyBlock(ZSTDMT_jobDescription* job) +{ + assert(job->lastJob == 1); + assert(job->src.size == 0); /* last job is empty -> will be simplified into a last empty block */ + assert(job->firstJob == 0); /* cannot be first job, as it also needs to create frame header */ + assert(job->dstBuff.start == NULL); /* invoked from streaming variant only (otherwise, dstBuff might be user's output) */ + job->dstBuff = ZSTDMT_getBuffer(job->bufPool); + if (job->dstBuff.start == NULL) { + job->cSize = ERROR(memory_allocation); + return; + } + assert(job->dstBuff.capacity >= ZSTD_blockHeaderSize); /* no buffer should ever be that small */ + job->src = kNullRange; + job->cSize = ZSTD_writeLastEmptyBlock(job->dstBuff.start, job->dstBuff.capacity); + assert(!ZSTD_isError(job->cSize)); + assert(job->consumed == 0); +} + +static size_t ZSTDMT_createCompressionJob(ZSTDMT_CCtx* mtctx, size_t srcSize, ZSTD_EndDirective endOp) +{ + unsigned const jobID = mtctx->nextJobID & mtctx->jobIDMask; + int const endFrame = (endOp == ZSTD_e_end); + + if (mtctx->nextJobID > mtctx->doneJobID + mtctx->jobIDMask) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: will not create new job : table is full"); + assert((mtctx->nextJobID & mtctx->jobIDMask) == (mtctx->doneJobID & mtctx->jobIDMask)); + return 0; + } + + if (!mtctx->jobReady) { + BYTE const* src = (BYTE const*)mtctx->inBuff.buffer.start; + DEBUGLOG(5, "ZSTDMT_createCompressionJob: preparing job %u to compress %u bytes with %u preload ", + mtctx->nextJobID, (U32)srcSize, (U32)mtctx->inBuff.prefix.size); + mtctx->jobs[jobID].src.start = src; + mtctx->jobs[jobID].src.size = srcSize; + assert(mtctx->inBuff.filled >= srcSize); + mtctx->jobs[jobID].prefix = mtctx->inBuff.prefix; + mtctx->jobs[jobID].consumed = 0; + mtctx->jobs[jobID].cSize = 0; + mtctx->jobs[jobID].params = mtctx->params; + mtctx->jobs[jobID].cdict = mtctx->nextJobID==0 ? mtctx->cdict : NULL; + mtctx->jobs[jobID].fullFrameSize = mtctx->frameContentSize; + mtctx->jobs[jobID].dstBuff = g_nullBuffer; + mtctx->jobs[jobID].cctxPool = mtctx->cctxPool; + mtctx->jobs[jobID].bufPool = mtctx->bufPool; + mtctx->jobs[jobID].seqPool = mtctx->seqPool; + mtctx->jobs[jobID].serial = &mtctx->serial; + mtctx->jobs[jobID].jobID = mtctx->nextJobID; + mtctx->jobs[jobID].firstJob = (mtctx->nextJobID==0); + mtctx->jobs[jobID].lastJob = endFrame; + mtctx->jobs[jobID].frameChecksumNeeded = mtctx->params.fParams.checksumFlag && endFrame && (mtctx->nextJobID>0); + mtctx->jobs[jobID].dstFlushed = 0; + + /* Update the round buffer pos and clear the input buffer to be reset */ + mtctx->roundBuff.pos += srcSize; + mtctx->inBuff.buffer = g_nullBuffer; + mtctx->inBuff.filled = 0; + /* Set the prefix */ + if (!endFrame) { + size_t const newPrefixSize = MIN(srcSize, mtctx->targetPrefixSize); + mtctx->inBuff.prefix.start = src + srcSize - newPrefixSize; + mtctx->inBuff.prefix.size = newPrefixSize; + } else { /* endFrame==1 => no need for another input buffer */ + mtctx->inBuff.prefix = kNullRange; + mtctx->frameEnded = endFrame; + if (mtctx->nextJobID == 0) { + /* single job exception : checksum is already calculated directly within worker thread */ + mtctx->params.fParams.checksumFlag = 0; + } } + + if ( (srcSize == 0) + && (mtctx->nextJobID>0)/*single job must also write frame header*/ ) { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: creating a last empty block to end frame"); + assert(endOp == ZSTD_e_end); /* only possible case : need to end the frame with an empty last block */ + ZSTDMT_writeLastEmptyBlock(mtctx->jobs + jobID); + mtctx->nextJobID++; + return 0; + } + } + + DEBUGLOG(5, "ZSTDMT_createCompressionJob: posting job %u : %u bytes (end:%u, jobNb == %u (mod:%u))", + mtctx->nextJobID, + (U32)mtctx->jobs[jobID].src.size, + mtctx->jobs[jobID].lastJob, + mtctx->nextJobID, + jobID); + if (POOL_tryAdd(mtctx->factory, ZSTDMT_compressionJob, &mtctx->jobs[jobID])) { + mtctx->nextJobID++; + mtctx->jobReady = 0; + } else { + DEBUGLOG(5, "ZSTDMT_createCompressionJob: no worker available for job %u", mtctx->nextJobID); + mtctx->jobReady = 1; + } + return 0; +} + + +/*! ZSTDMT_flushProduced() : + * flush whatever data has been produced but not yet flushed in current job. + * move to next job if current one is fully flushed. + * `output` : `pos` will be updated with amount of data flushed . + * `blockToFlush` : if >0, the function will block and wait if there is no data available to flush . + * @return : amount of data remaining within internal buffer, 0 if no more, 1 if unknown but > 0, or an error code */ +static size_t ZSTDMT_flushProduced(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, unsigned blockToFlush, ZSTD_EndDirective end) +{ + unsigned const wJobID = mtctx->doneJobID & mtctx->jobIDMask; + DEBUGLOG(5, "ZSTDMT_flushProduced (blocking:%u , job %u <= %u)", + blockToFlush, mtctx->doneJobID, mtctx->nextJobID); + assert(output->size >= output->pos); + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + if ( blockToFlush + && (mtctx->doneJobID < mtctx->nextJobID) ) { + assert(mtctx->jobs[wJobID].dstFlushed <= mtctx->jobs[wJobID].cSize); + while (mtctx->jobs[wJobID].dstFlushed == mtctx->jobs[wJobID].cSize) { /* nothing to flush */ + if (mtctx->jobs[wJobID].consumed == mtctx->jobs[wJobID].src.size) { + DEBUGLOG(5, "job %u is completely consumed (%u == %u) => don't wait for cond, there will be none", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].consumed, (U32)mtctx->jobs[wJobID].src.size); + break; + } + DEBUGLOG(5, "waiting for something to flush from job %u (currently flushed: %u bytes)", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTD_pthread_cond_wait(&mtctx->jobs[wJobID].job_cond, &mtctx->jobs[wJobID].job_mutex); /* block when nothing to flush but some to come */ + } } + + /* try to flush something */ + { size_t cSize = mtctx->jobs[wJobID].cSize; /* shared */ + size_t const srcConsumed = mtctx->jobs[wJobID].consumed; /* shared */ + size_t const srcSize = mtctx->jobs[wJobID].src.size; /* read-only, could be done after mutex lock, but no-declaration-after-statement */ + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + if (ZSTD_isError(cSize)) { + DEBUGLOG(5, "ZSTDMT_flushProduced: job %u : compression error detected : %s", + mtctx->doneJobID, ZSTD_getErrorName(cSize)); + ZSTDMT_waitForAllJobsCompleted(mtctx); + ZSTDMT_releaseAllJobResources(mtctx); + return cSize; + } + /* add frame checksum if necessary (can only happen once) */ + assert(srcConsumed <= srcSize); + if ( (srcConsumed == srcSize) /* job completed -> worker no longer active */ + && mtctx->jobs[wJobID].frameChecksumNeeded ) { + U32 const checksum = (U32)XXH64_digest(&mtctx->serial.xxhState); + DEBUGLOG(4, "ZSTDMT_flushProduced: writing checksum : %08X \n", checksum); + MEM_writeLE32((char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].cSize, checksum); + cSize += 4; + mtctx->jobs[wJobID].cSize += 4; /* can write this shared value, as worker is no longer active */ + mtctx->jobs[wJobID].frameChecksumNeeded = 0; + } + + if (cSize > 0) { /* compression is ongoing or completed */ + size_t const toFlush = MIN(cSize - mtctx->jobs[wJobID].dstFlushed, output->size - output->pos); + DEBUGLOG(5, "ZSTDMT_flushProduced: Flushing %u bytes from job %u (completion:%u/%u, generated:%u)", + (U32)toFlush, mtctx->doneJobID, (U32)srcConsumed, (U32)srcSize, (U32)cSize); + assert(mtctx->doneJobID < mtctx->nextJobID); + assert(cSize >= mtctx->jobs[wJobID].dstFlushed); + assert(mtctx->jobs[wJobID].dstBuff.start != NULL); + if (toFlush > 0) { + ZSTD_memcpy((char*)output->dst + output->pos, + (const char*)mtctx->jobs[wJobID].dstBuff.start + mtctx->jobs[wJobID].dstFlushed, + toFlush); + } + output->pos += toFlush; + mtctx->jobs[wJobID].dstFlushed += toFlush; /* can write : this value is only used by mtctx */ + + if ( (srcConsumed == srcSize) /* job is completed */ + && (mtctx->jobs[wJobID].dstFlushed == cSize) ) { /* output buffer fully flushed => free this job position */ + DEBUGLOG(5, "Job %u completed (%u bytes), moving to next one", + mtctx->doneJobID, (U32)mtctx->jobs[wJobID].dstFlushed); + ZSTDMT_releaseBuffer(mtctx->bufPool, mtctx->jobs[wJobID].dstBuff); + DEBUGLOG(5, "dstBuffer released"); + mtctx->jobs[wJobID].dstBuff = g_nullBuffer; + mtctx->jobs[wJobID].cSize = 0; /* ensure this job slot is considered "not started" in future check */ + mtctx->consumed += srcSize; + mtctx->produced += cSize; + mtctx->doneJobID++; + } } + + /* return value : how many bytes left in buffer ; fake it to 1 when unknown but >0 */ + if (cSize > mtctx->jobs[wJobID].dstFlushed) return (cSize - mtctx->jobs[wJobID].dstFlushed); + if (srcSize > srcConsumed) return 1; /* current job not completely compressed */ + } + if (mtctx->doneJobID < mtctx->nextJobID) return 1; /* some more jobs ongoing */ + if (mtctx->jobReady) return 1; /* one job is ready to push, just not yet in the list */ + if (mtctx->inBuff.filled > 0) return 1; /* input is not empty, and still needs to be converted into a job */ + mtctx->allJobsCompleted = mtctx->frameEnded; /* all jobs are entirely flushed => if this one is last one, frame is completed */ + if (end == ZSTD_e_end) return !mtctx->frameEnded; /* for ZSTD_e_end, question becomes : is frame completed ? instead of : are internal buffers fully flushed ? */ + return 0; /* internal buffers fully flushed */ +} + +/** + * Returns the range of data used by the earliest job that is not yet complete. + * If the data of the first job is broken up into two segments, we cover both + * sections. + */ +static range_t ZSTDMT_getInputDataInUse(ZSTDMT_CCtx* mtctx) +{ + unsigned const firstJobID = mtctx->doneJobID; + unsigned const lastJobID = mtctx->nextJobID; + unsigned jobID; + + for (jobID = firstJobID; jobID < lastJobID; ++jobID) { + unsigned const wJobID = jobID & mtctx->jobIDMask; + size_t consumed; + + ZSTD_PTHREAD_MUTEX_LOCK(&mtctx->jobs[wJobID].job_mutex); + consumed = mtctx->jobs[wJobID].consumed; + ZSTD_pthread_mutex_unlock(&mtctx->jobs[wJobID].job_mutex); + + if (consumed < mtctx->jobs[wJobID].src.size) { + range_t range = mtctx->jobs[wJobID].prefix; + if (range.size == 0) { + /* Empty prefix */ + range = mtctx->jobs[wJobID].src; + } + /* Job source in multiple segments not supported yet */ + assert(range.start <= mtctx->jobs[wJobID].src.start); + return range; + } + } + return kNullRange; +} + +/** + * Returns non-zero iff buffer and range overlap. + */ +static int ZSTDMT_isOverlapped(buffer_t buffer, range_t range) +{ + BYTE const* const bufferStart = (BYTE const*)buffer.start; + BYTE const* const rangeStart = (BYTE const*)range.start; + + if (rangeStart == NULL || bufferStart == NULL) + return 0; + + { + BYTE const* const bufferEnd = bufferStart + buffer.capacity; + BYTE const* const rangeEnd = rangeStart + range.size; + + /* Empty ranges cannot overlap */ + if (bufferStart == bufferEnd || rangeStart == rangeEnd) + return 0; + + return bufferStart < rangeEnd && rangeStart < bufferEnd; + } +} + +static int ZSTDMT_doesOverlapWindow(buffer_t buffer, ZSTD_window_t window) +{ + range_t extDict; + range_t prefix; + + DEBUGLOG(5, "ZSTDMT_doesOverlapWindow"); + extDict.start = window.dictBase + window.lowLimit; + extDict.size = window.dictLimit - window.lowLimit; + + prefix.start = window.base + window.dictLimit; + prefix.size = window.nextSrc - (window.base + window.dictLimit); + DEBUGLOG(5, "extDict [0x%zx, 0x%zx)", + (size_t)extDict.start, + (size_t)extDict.start + extDict.size); + DEBUGLOG(5, "prefix [0x%zx, 0x%zx)", + (size_t)prefix.start, + (size_t)prefix.start + prefix.size); + + return ZSTDMT_isOverlapped(buffer, extDict) + || ZSTDMT_isOverlapped(buffer, prefix); +} + +static void ZSTDMT_waitForLdmComplete(ZSTDMT_CCtx* mtctx, buffer_t buffer) +{ + if (mtctx->params.ldmParams.enableLdm == ZSTD_ps_enable) { + ZSTD_pthread_mutex_t* mutex = &mtctx->serial.ldmWindowMutex; + DEBUGLOG(5, "ZSTDMT_waitForLdmComplete"); + DEBUGLOG(5, "source [0x%zx, 0x%zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + ZSTD_PTHREAD_MUTEX_LOCK(mutex); + while (ZSTDMT_doesOverlapWindow(buffer, mtctx->serial.ldmWindow)) { + DEBUGLOG(5, "Waiting for LDM to finish..."); + ZSTD_pthread_cond_wait(&mtctx->serial.ldmWindowCond, mutex); + } + DEBUGLOG(6, "Done waiting for LDM to finish"); + ZSTD_pthread_mutex_unlock(mutex); + } +} + +/** + * Attempts to set the inBuff to the next section to fill. + * If any part of the new section is still in use we give up. + * Returns non-zero if the buffer is filled. + */ +static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) +{ + range_t const inUse = ZSTDMT_getInputDataInUse(mtctx); + size_t const spaceLeft = mtctx->roundBuff.capacity - mtctx->roundBuff.pos; + size_t const target = mtctx->targetSectionSize; + buffer_t buffer; + + DEBUGLOG(5, "ZSTDMT_tryGetInputRange"); + assert(mtctx->inBuff.buffer.start == NULL); + assert(mtctx->roundBuff.capacity >= target); + + if (spaceLeft < target) { + /* ZSTD_invalidateRepCodes() doesn't work for extDict variants. + * Simply copy the prefix to the beginning in that case. + */ + BYTE* const start = (BYTE*)mtctx->roundBuff.buffer; + size_t const prefixSize = mtctx->inBuff.prefix.size; + + buffer.start = start; + buffer.capacity = prefixSize; + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + ZSTDMT_waitForLdmComplete(mtctx, buffer); + ZSTD_memmove(start, mtctx->inBuff.prefix.start, prefixSize); + mtctx->inBuff.prefix.start = start; + mtctx->roundBuff.pos = prefixSize; + } + buffer.start = mtctx->roundBuff.buffer + mtctx->roundBuff.pos; + buffer.capacity = target; + + if (ZSTDMT_isOverlapped(buffer, inUse)) { + DEBUGLOG(5, "Waiting for buffer..."); + return 0; + } + assert(!ZSTDMT_isOverlapped(buffer, mtctx->inBuff.prefix)); + + ZSTDMT_waitForLdmComplete(mtctx, buffer); + + DEBUGLOG(5, "Using prefix range [%zx, %zx)", + (size_t)mtctx->inBuff.prefix.start, + (size_t)mtctx->inBuff.prefix.start + mtctx->inBuff.prefix.size); + DEBUGLOG(5, "Using source range [%zx, %zx)", + (size_t)buffer.start, + (size_t)buffer.start + buffer.capacity); + + + mtctx->inBuff.buffer = buffer; + mtctx->inBuff.filled = 0; + assert(mtctx->roundBuff.pos + buffer.capacity <= mtctx->roundBuff.capacity); + return 1; +} + +typedef struct { + size_t toLoad; /* The number of bytes to load from the input. */ + int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ +} syncPoint_t; + +/** + * Searches through the input for a synchronization point. If one is found, we + * will instruct the caller to flush, and return the number of bytes to load. + * Otherwise, we will load as many bytes as possible and instruct the caller + * to continue as normal. + */ +static syncPoint_t +findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) +{ + BYTE const* const istart = (BYTE const*)input.src + input.pos; + U64 const primePower = mtctx->rsync.primePower; + U64 const hitMask = mtctx->rsync.hitMask; + + syncPoint_t syncPoint; + U64 hash; + BYTE const* prev; + size_t pos; + + syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled); + syncPoint.flush = 0; + if (!mtctx->params.rsyncable) + /* Rsync is disabled. */ + return syncPoint; + if (mtctx->inBuff.filled + input.size - input.pos < RSYNC_MIN_BLOCK_SIZE) + /* We don't emit synchronization points if it would produce too small blocks. + * We don't have enough input to find a synchronization point, so don't look. + */ + return syncPoint; + if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) + /* Not enough to compute the hash. + * We will miss any synchronization points in this RSYNC_LENGTH byte + * window. However, since it depends only in the internal buffers, if the + * state is already synchronized, we will remain synchronized. + * Additionally, the probability that we miss a synchronization point is + * low: RSYNC_LENGTH / targetSectionSize. + */ + return syncPoint; + /* Initialize the loop variables. */ + if (mtctx->inBuff.filled < RSYNC_MIN_BLOCK_SIZE) { + /* We don't need to scan the first RSYNC_MIN_BLOCK_SIZE positions + * because they can't possibly be a sync point. So we can start + * part way through the input buffer. + */ + pos = RSYNC_MIN_BLOCK_SIZE - mtctx->inBuff.filled; + if (pos >= RSYNC_LENGTH) { + prev = istart + pos - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + } else { + assert(mtctx->inBuff.filled >= RSYNC_LENGTH); + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev + pos, (RSYNC_LENGTH - pos)); + hash = ZSTD_rollingHash_append(hash, istart, pos); + } + } else { + /* We have enough bytes buffered to initialize the hash, + * and have processed enough bytes to find a sync point. + * Start scanning at the beginning of the input. + */ + assert(mtctx->inBuff.filled >= RSYNC_MIN_BLOCK_SIZE); + assert(RSYNC_MIN_BLOCK_SIZE >= RSYNC_LENGTH); + pos = 0; + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + if ((hash & hitMask) == hitMask) { + /* We're already at a sync point so don't load any more until + * we're able to flush this sync point. + * This likely happened because the job table was full so we + * couldn't add our job. + */ + syncPoint.toLoad = 0; + syncPoint.flush = 1; + return syncPoint; + } + } + /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll + * through the input. If we hit a synchronization point, then cut the + * job off, and tell the compressor to flush the job. Otherwise, load + * all the bytes and continue as normal. + * If we go too long without a synchronization point (targetSectionSize) + * then a block will be emitted anyways, but this is okay, since if we + * are already synchronized we will remain synchronized. + */ + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + for (; pos < syncPoint.toLoad; ++pos) { + BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; + /* This assert is very expensive, and Debian compiles with asserts enabled. + * So disable it for now. We can get similar coverage by checking it at the + * beginning & end of the loop. + * assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + */ + hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); + assert(mtctx->inBuff.filled + pos >= RSYNC_MIN_BLOCK_SIZE); + if ((hash & hitMask) == hitMask) { + syncPoint.toLoad = pos + 1; + syncPoint.flush = 1; + ++pos; /* for assert */ + break; + } + } + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + return syncPoint; +} + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx) +{ + size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled; + if (hintInSize==0) hintInSize = mtctx->targetSectionSize; + return hintInSize; +} + +/** ZSTDMT_compressStream_generic() : + * internal use only - exposed to be invoked from zstd_compress.c + * assumption : output and input are valid (pos <= size) + * @return : minimum amount of data remaining to flush, 0 if none */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) +{ + unsigned forwardInputProgress = 0; + DEBUGLOG(5, "ZSTDMT_compressStream_generic (endOp=%u, srcSize=%u)", + (U32)endOp, (U32)(input->size - input->pos)); + assert(output->pos <= output->size); + assert(input->pos <= input->size); + + if ((mtctx->frameEnded) && (endOp==ZSTD_e_continue)) { + /* current frame being ended. Only flush/end are allowed */ + return ERROR(stage_wrong); + } + + /* fill input buffer */ + if ( (!mtctx->jobReady) + && (input->size > input->pos) ) { /* support NULL input */ + if (mtctx->inBuff.buffer.start == NULL) { + assert(mtctx->inBuff.filled == 0); /* Can't fill an empty buffer */ + if (!ZSTDMT_tryGetInputRange(mtctx)) { + /* It is only possible for this operation to fail if there are + * still compression jobs ongoing. + */ + DEBUGLOG(5, "ZSTDMT_tryGetInputRange failed"); + assert(mtctx->doneJobID != mtctx->nextJobID); + } else + DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); + } + if (mtctx->inBuff.buffer.start != NULL) { + syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input); + if (syncPoint.flush && endOp == ZSTD_e_continue) { + endOp = ZSTD_e_flush; + } + assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); + DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", + (U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); + ZSTD_memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); + input->pos += syncPoint.toLoad; + mtctx->inBuff.filled += syncPoint.toLoad; + forwardInputProgress = syncPoint.toLoad>0; + } + } + if ((input->pos < input->size) && (endOp == ZSTD_e_end)) { + /* Can't end yet because the input is not fully consumed. + * We are in one of these cases: + * - mtctx->inBuff is NULL & empty: we couldn't get an input buffer so don't create a new job. + * - We filled the input buffer: flush this job but don't end the frame. + * - We hit a synchronization point: flush this job but don't end the frame. + */ + assert(mtctx->inBuff.filled == 0 || mtctx->inBuff.filled == mtctx->targetSectionSize || mtctx->params.rsyncable); + endOp = ZSTD_e_flush; + } + + if ( (mtctx->jobReady) + || (mtctx->inBuff.filled >= mtctx->targetSectionSize) /* filled enough : let's compress */ + || ((endOp != ZSTD_e_continue) && (mtctx->inBuff.filled > 0)) /* something to flush : let's go */ + || ((endOp == ZSTD_e_end) && (!mtctx->frameEnded)) ) { /* must finish the frame with a zero-size block */ + size_t const jobSize = mtctx->inBuff.filled; + assert(mtctx->inBuff.filled <= mtctx->targetSectionSize); + FORWARD_IF_ERROR( ZSTDMT_createCompressionJob(mtctx, jobSize, endOp) , ""); + } + + /* check for potential compressed data ready to be flushed */ + { size_t const remainingToFlush = ZSTDMT_flushProduced(mtctx, output, !forwardInputProgress, endOp); /* block if there was no forward input progress */ + if (input->pos < input->size) return MAX(remainingToFlush, 1); /* input not consumed : do not end flush yet */ + DEBUGLOG(5, "end of ZSTDMT_compressStream_generic: remainingToFlush = %u", (U32)remainingToFlush); + return remainingToFlush; + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.h b/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.h new file mode 100644 index 000000000..ed4dc0e99 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/compress/zstdmt_compress.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + #ifndef ZSTDMT_COMPRESS_H + #define ZSTDMT_COMPRESS_H + + #if defined (__cplusplus) + extern "C" { + #endif + + +/* Note : This is an internal API. + * These APIs used to be exposed with ZSTDLIB_API, + * because it used to be the only way to invoke MT compression. + * Now, you must use ZSTD_compress2 and ZSTD_compressStream2() instead. + * + * This API requires ZSTD_MULTITHREAD to be defined during compilation, + * otherwise ZSTDMT_createCCtx*() will fail. + */ + +/* === Dependencies === */ +#include "../common/zstd_deps.h" /* size_t */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters */ +#include "../zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ + + +/* === Constants === */ +#ifndef ZSTDMT_NBWORKERS_MAX /* a different value can be selected at compile time */ +# define ZSTDMT_NBWORKERS_MAX ((sizeof(void*)==4) /*32-bit*/ ? 64 : 256) +#endif +#ifndef ZSTDMT_JOBSIZE_MIN /* a different value can be selected at compile time */ +# define ZSTDMT_JOBSIZE_MIN (512 KB) +#endif +#define ZSTDMT_JOBLOG_MAX (MEM_32bits() ? 29 : 30) +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) + + +/* ======================================================== + * === Private interface, for use by ZSTD_compress.c === + * === Not exposed in libzstd. Never invoke directly === + * ======================================================== */ + +/* === Memory management === */ +typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; +/* Requires ZSTD_MULTITHREAD to be defined during compilation, otherwise it will return NULL. */ +ZSTDMT_CCtx* ZSTDMT_createCCtx_advanced(unsigned nbWorkers, + ZSTD_customMem cMem, + ZSTD_threadPool *pool); +size_t ZSTDMT_freeCCtx(ZSTDMT_CCtx* mtctx); + +size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx); + +/* === Streaming functions === */ + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_initCStream_internal() : + * Private use only. Init streaming operation. + * expects params to be valid. + * must receive dict, or cdict, or none, but not both. + * mtctx can be freshly constructed or reused from a prior compression. + * If mtctx is reused, memory allocations from the prior compression may not be freed, + * even if they are not needed for the current compression. + * @return : 0, or an error code */ +size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* mtctx, + const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + +/*! ZSTDMT_compressStream_generic() : + * Combines ZSTDMT_compressStream() with optional ZSTDMT_flushStream() or ZSTDMT_endStream() + * depending on flush directive. + * @return : minimum amount of data still to be flushed + * 0 if fully flushed + * or an error code + * note : needs to be init using any ZSTD_initCStream*() variant */ +size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + /*! ZSTDMT_toFlushNow() + * Tell how many bytes are ready to be flushed immediately. + * Probe the oldest active job (not yet entirely flushed) and check its output buffer. + * If return 0, it means there is no active job, + * or, it means oldest job is still active, but everything produced has been flushed so far, + * therefore flushing is limited by speed of oldest job. */ +size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); + +/*! ZSTDMT_updateCParams_whileCompressing() : + * Updates only a selected set of compression parameters, to remain compatible with current frame. + * New parameters will be applied to next compression job. */ +void ZSTDMT_updateCParams_whileCompressing(ZSTDMT_CCtx* mtctx, const ZSTD_CCtx_params* cctxParams); + +/*! ZSTDMT_getFrameProgression(): + * tells how much data has been consumed (input) and produced (output) for current frame. + * able to count progression inside worker threads. + */ +ZSTD_frameProgression ZSTDMT_getFrameProgression(ZSTDMT_CCtx* mtctx); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTDMT_COMPRESS_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress.c b/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress.c new file mode 100644 index 000000000..5b217ac58 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress.c @@ -0,0 +1,1882 @@ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" /* BIT_* */ +#include "../common/fse.h" /* to compress headers */ +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/zstd_internal.h" +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */ + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 + +/* ************************************************************** +* Macros +****************************************************************/ + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_FAST_BMI2_ATTRS +#endif + +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + if (flags & HUF_flags_bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + (void)flags; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilimit [in] - The input limit, stop when any input pointer is below ilimit. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilimit, but that indicates corruption. + */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilimit; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); + +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const ilimit = (const BYTE*)src + 6 + 8; + + BYTE* const oend = (BYTE*)dst + dstSize; + + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; + + /* Read the jump table. */ + { + const BYTE* const istart = (const BYTE*)src; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + * length1 must be >= 16 so that ip[0] >= ilimit before the loop + * starts. + */ + if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* If ip[] >= ilimit, it is guaranteed to be safe to + * reload bits[]. It may be beyond its section, but is + * guaranteed to be valid (>= istart). + */ + args->ilimit = ilimit; + + args->oend = oend; + args->dt = dt; + + return 1; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->iend[0]; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ + +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (U64)((symbol << 8) + nbBits); + } else { + D4 = (U64)(symbol + (nbBits << 8)); + } + assert(D4 < (1U << 16)); + D4 *= 0x0001000100010001ULL; + return D4; +} + +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { int n; + U32 nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return (size_t)(pEnd-pStart); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilimit = args->ilimit; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + int symbol; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilimit); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilimit) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we get close to the end. */ + if (op[3] + 20 > olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + + do { + /* Decode 5 symbols in each of the 4 streams */ + for (symbol = 0; symbol < 5; ++symbol) { + for (stream = 0; stream < 4; ++stream) { + int const index = (int)(bits[stream] >> 53); + int const entry = (int)dtable[index]; + bits[stream] <<= (entry & 63); + op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF); + } + } + /* Reload the bitstreams */ + for (stream = 0; stream < 4; ++stream) { + int const ctz = ZSTD_countTrailingZeros64(bits[stream]); + int const nbBits = ctz & 7; + int const nbBytes = ctz >> 3; + op[stream] += 5; + ip[stream] -= nbBytes; + bits[stream] = MEM_read64(ip[stream]) | 1; + bits[stream] <<= nbBits; + } + } while (op[3] < olimit); + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilimit); + loopFn(&args); + + /* Our loop guarantees that ip[] >= ilimit and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) + +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +#endif + + if (!(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilimit = args->ilimit; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + int symbol; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilimit); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilimit) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + + /* Exit the fast decoding loop if we are too close to the end. */ + if (op[3] + 10 > olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + + do { + /* Do 5 table lookups for each of the first 3 streams */ + for (symbol = 0; symbol < 5; ++symbol) { + for (stream = 0; stream < 3; ++stream) { + int const index = (int)(bits[stream] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[stream], entry.sequence); + bits[stream] <<= (entry.nbBits); + op[stream] += (entry.length); + } + } + /* Do 1 table lookup from the final stream */ + { + int const index = (int)(bits[3] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[3], entry.sequence); + bits[3] <<= (entry.nbBits); + op[3] += (entry.length); + } + /* Do 4 table lookups from the final stream & reload bitstreams */ + for (stream = 0; stream < 4; ++stream) { + /* Do a table lookup from the final stream. + * This is interleaved with the reloading to reduce register + * pressure. This shouldn't be necessary, but compilers can + * struggle with codegen with high register pressure. + */ + { + int const index = (int)(bits[3] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[3], entry.sequence); + bits[3] <<= (entry.nbBits); + op[3] += (entry.length); + } + /* Reload the bistreams. The final bitstream must be reloaded + * after the 5th symbol was decoded. + */ + { + int const ctz = ZSTD_countTrailingZeros64(bits[stream]); + int const nbBits = ctz & 7; + int const nbBytes = ctz >> 3; + ip[stream] -= nbBytes; + bits[stream] = MEM_read64(ip[stream]) | 1; + bits[stream] <<= nbBits; + } + } + } while (op[3] < olimit); + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const iend = (const BYTE*)cSrc + 6; + BYTE* const oend = (BYTE*)dst + dstSize; + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilimit); + loopFn(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= iend); + assert(args.ip[1] >= iend); + assert(args.ip[2] >= iend); + assert(args.ip[3] >= iend); + assert(args.op[3] <= oend); + (void)iend; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} + +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif + + if (!(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); +} + +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} +#endif + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#endif + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress_amd64.S b/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress_amd64.S new file mode 100644 index 000000000..671624fe3 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/huf_decompress_amd64.S @@ -0,0 +1,576 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "../common/portability_macros.h" + +/* Stack marking + * ref: https://wiki.gentoo.org/wiki/Hardened/GNU_stack_quickstart + */ +#if defined(__ELF__) && defined(__GNUC__) +.section .note.GNU-stack,"",%progbits +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +/* Calling convention: + * + * %rdi contains the first argument: HUF_DecompressAsmArgs*. + * %rbp isn't maintained (no frame pointer). + * %rsp contains the stack pointer that grows down. + * No red-zone is assumed, only addresses >= %rsp are used. + * All register contents are preserved. + * + * TODO: Support Windows calling convention. + */ + +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +.global HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.text + +/* Sets up register mappings for clarity. + * op[], bits[], dtable & ip[0] each get their own register. + * ip[1,2,3] & olimit alias var[]. + * %rax is a scratch register. + */ + +#define op0 rsi +#define op1 rbx +#define op2 rcx +#define op3 rdi + +#define ip0 r8 +#define ip1 r9 +#define ip2 r10 +#define ip3 r11 + +#define bits0 rbp +#define bits1 rdx +#define bits2 r12 +#define bits3 r13 +#define dtable r14 +#define olimit r15 + +/* var[] aliases ip[1,2,3] & olimit + * ip[1,2,3] are saved every iteration. + * olimit is only used in compute_olimit. + */ +#define var0 r15 +#define var1 r9 +#define var2 r10 +#define var3 r11 + +/* 32-bit var registers */ +#define vard0 r15d +#define vard1 r9d +#define vard2 r10d +#define vard3 r11d + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM(X) \ + X(0); \ + X(1); \ + X(2); \ + X(3) + +/* Calls X(N, idx) for each stream 0, 1, 2, 3. */ +#define FOR_EACH_STREAM_WITH_INDEX(X, idx) \ + X(0, idx); \ + X(1, idx); \ + X(2, idx); \ + X(3, idx) + +/* Define both _HUF_* & HUF_* symbols because MacOS + * C symbols are prefixed with '_' & Linux symbols aren't. + */ +_HUF_decompress4X1_usingDTable_internal_fast_asm_loop: +HUF_decompress4X1_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + /* Read HUF_DecompressAsmArgs* args from %rax */ + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push 104(%rax) /* ilimit */ + push 112(%rax) /* oend */ + push %olimit /* olimit space */ + + subq $24, %rsp + +.L_4X1_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rbx, rdx must be saved + * op3 & ip0 mustn't be clobbered + */ + movq %rbx, 0(%rsp) + movq %rdx, 8(%rsp) + + movq 32(%rsp), %rax /* rax = oend */ + subq %op3, %rax /* rax = oend - op3 */ + + /* r15 = (oend - op3) / 5 */ + movabsq $-3689348814741910323, %rdx + mulq %rdx + movq %rdx, %r15 + shrq $2, %r15 + + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilimit */ + subq %rdx, %rax /* rax = ip0 - ilimit */ + movq %rax, %rbx /* rbx = ip0 - ilimit */ + + /* rdx = (ip0 - ilimit) / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %rbx + shrq %rbx + addq %rbx, %rdx + shrq $2, %rdx + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* r15 = r15 * 5 */ + leaq (%r15, %r15, 4), %r15 + + /* olimit = op3 + r15 */ + addq %op3, %olimit + + movq 8(%rsp), %rdx + movq 0(%rsp), %rbx + + /* If (op3 + 20 > olimit) */ + movq %op3, %rax /* rax = op3 */ + addq $20, %rax /* rax = op3 + 20 */ + cmpq %rax, %olimit /* op3 + 20 > olimit */ + jb .L_4X1_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X1_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X1_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X1_exit + +/* Reads top 11 bits from bits[n] + * Loads dt[bits[n]] into var[n] + */ +#define GET_NEXT_DELT(n) \ + movq $53, %var##n; \ + shrxq %var##n, %bits##n, %var##n; \ + movzwl (%dtable,%var##n,2),%vard##n + +/* var[n] must contain the DTable entry computed with GET_NEXT_DELT + * Moves var[n] to %rax + * bits[n] <<= var[n] & 63 + * op[n][idx] = %rax >> 8 + * %ah is a way to access bits [8, 16) of %rax + */ +#define DECODE_FROM_DELT(n, idx) \ + movq %var##n, %rax; \ + shlxq %var##n, %bits##n, %bits##n; \ + movb %ah, idx(%op##n) + +/* Assumes GET_NEXT_DELT has been called. + * Calls DECODE_FROM_DELT then GET_NEXT_DELT + */ +#define DECODE_AND_GET_NEXT(n, idx) \ + DECODE_FROM_DELT(n, idx); \ + GET_NEXT_DELT(n) \ + +/* // ctz & nbBytes is stored in bits[n] + * // nbBits is stored in %rax + * ctz = CTZ[bits[n]] + * nbBits = ctz & 7 + * nbBytes = ctz >> 3 + * op[n] += 5 + * ip[n] -= nbBytes + * // Note: x86-64 is little-endian ==> no bswap + * bits[n] = MEM_readST(ip[n]) | 1 + * bits[n] <<= nbBits + */ +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + andq $7, %rax; \ + shrq $3, %bits##n; \ + leaq 5(%op##n), %op##n; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlx %rax, %bits##n, %bits##n + + /* Store clobbered variables on the stack */ + movq %olimit, 24(%rsp) + movq %ip1, 0(%rsp) + movq %ip2, 8(%rsp) + movq %ip3, 16(%rsp) + + /* Call GET_NEXT_DELT for each stream */ + FOR_EACH_STREAM(GET_NEXT_DELT) + + .p2align 6 + +.L_4X1_loop_body: + /* Decode 5 symbols in each of the 4 streams (20 total) + * Must have called GET_NEXT_DELT for each stream + */ + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE_AND_GET_NEXT, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE_FROM_DELT, 4) + + /* Load ip[1,2,3] from stack (var[] aliases them) + * ip[] is needed for RELOAD_BITS + * Each will be stored back to the stack after RELOAD + */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Reload each stream & fetch the next table entry + * to prepare for the next iteration + */ + RELOAD_BITS(0) + GET_NEXT_DELT(0) + + RELOAD_BITS(1) + movq %ip1, 0(%rsp) + GET_NEXT_DELT(1) + + RELOAD_BITS(2) + movq %ip2, 8(%rsp) + GET_NEXT_DELT(2) + + RELOAD_BITS(3) + movq %ip3, 16(%rsp) + GET_NEXT_DELT(3) + + /* If op3 < olimit: continue the loop */ + cmp %op3, 24(%rsp) + ja .L_4X1_loop_body + + /* Reload ip[1,2,3] from stack */ + movq 0(%rsp), %ip1 + movq 8(%rsp), %ip2 + movq 16(%rsp), %ip3 + + /* Re-compute olimit */ + jmp .L_4X1_compute_olimit + +#undef GET_NEXT_DELT +#undef DECODE_FROM_DELT +#undef DECODE +#undef RELOAD_BITS +.L_4X1_exit: + addq $24, %rsp + + /* Restore stack (oend & olimit) */ + pop %rax /* olimit */ + pop %rax /* oend */ + pop %rax /* ilimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +_HUF_decompress4X2_usingDTable_internal_fast_asm_loop: +HUF_decompress4X2_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH + /* Save all registers - even if they are callee saved for simplicity. */ + push %rax + push %rbx + push %rcx + push %rdx + push %rbp + push %rsi + push %rdi + push %r8 + push %r9 + push %r10 + push %r11 + push %r12 + push %r13 + push %r14 + push %r15 + + movq %rdi, %rax + movq 0(%rax), %ip0 + movq 8(%rax), %ip1 + movq 16(%rax), %ip2 + movq 24(%rax), %ip3 + movq 32(%rax), %op0 + movq 40(%rax), %op1 + movq 48(%rax), %op2 + movq 56(%rax), %op3 + movq 64(%rax), %bits0 + movq 72(%rax), %bits1 + movq 80(%rax), %bits2 + movq 88(%rax), %bits3 + movq 96(%rax), %dtable + push %rax /* argument */ + push %rax /* olimit */ + push 104(%rax) /* ilimit */ + + movq 112(%rax), %rax + push %rax /* oend3 */ + + movq %op3, %rax + push %rax /* oend2 */ + + movq %op2, %rax + push %rax /* oend1 */ + + movq %op1, %rax + push %rax /* oend0 */ + + /* Scratch space */ + subq $8, %rsp + +.L_4X2_compute_olimit: + /* Computes how many iterations we can do safely + * %r15, %rax may be clobbered + * rdx must be saved + * op[1,2,3,4] & ip0 mustn't be clobbered + */ + movq %rdx, 0(%rsp) + + /* We can consume up to 7 input bytes each iteration. */ + movq %ip0, %rax /* rax = ip0 */ + movq 40(%rsp), %rdx /* rdx = ilimit */ + subq %rdx, %rax /* rax = ip0 - ilimit */ + movq %rax, %r15 /* r15 = ip0 - ilimit */ + + /* rdx = rax / 7 */ + movabsq $2635249153387078803, %rdx + mulq %rdx + subq %rdx, %r15 + shrq %r15 + addq %r15, %rdx + shrq $2, %rdx + + /* r15 = (ip0 - ilimit) / 7 */ + movq %rdx, %r15 + + /* r15 = min(r15, min(oend0 - op0, oend1 - op1, oend2 - op2, oend3 - op3) / 10) */ + movq 8(%rsp), %rax /* rax = oend0 */ + subq %op0, %rax /* rax = oend0 - op0 */ + movq 16(%rsp), %rdx /* rdx = oend1 */ + subq %op1, %rdx /* rdx = oend1 - op1 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movq 24(%rsp), %rax /* rax = oend2 */ + subq %op2, %rax /* rax = oend2 - op2 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movq 32(%rsp), %rax /* rax = oend3 */ + subq %op3, %rax /* rax = oend3 - op3 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movabsq $-3689348814741910323, %rax + mulq %rdx + shrq $3, %rdx /* rdx = rdx / 10 */ + + /* r15 = min(%rdx, %r15) */ + cmpq %rdx, %r15 + cmova %rdx, %r15 + + /* olimit = op3 + 5 * r15 */ + movq %r15, %rax + leaq (%op3, %rax, 4), %olimit + addq %rax, %olimit + + movq 0(%rsp), %rdx + + /* If (op3 + 10 > olimit) */ + movq %op3, %rax /* rax = op3 */ + addq $10, %rax /* rax = op3 + 10 */ + cmpq %rax, %olimit /* op3 + 10 > olimit */ + jb .L_4X2_exit + + /* If (ip1 < ip0) go to exit */ + cmpq %ip0, %ip1 + jb .L_4X2_exit + + /* If (ip2 < ip1) go to exit */ + cmpq %ip1, %ip2 + jb .L_4X2_exit + + /* If (ip3 < ip2) go to exit */ + cmpq %ip2, %ip3 + jb .L_4X2_exit + +#define DECODE(n, idx) \ + movq %bits##n, %rax; \ + shrq $53, %rax; \ + movzwl 0(%dtable,%rax,4),%r8d; \ + movzbl 2(%dtable,%rax,4),%r15d; \ + movzbl 3(%dtable,%rax,4),%eax; \ + movw %r8w, (%op##n); \ + shlxq %r15, %bits##n, %bits##n; \ + addq %rax, %op##n + +#define RELOAD_BITS(n) \ + bsfq %bits##n, %bits##n; \ + movq %bits##n, %rax; \ + shrq $3, %bits##n; \ + andq $7, %rax; \ + subq %bits##n, %ip##n; \ + movq (%ip##n), %bits##n; \ + orq $1, %bits##n; \ + shlxq %rax, %bits##n, %bits##n + + + movq %olimit, 48(%rsp) + + .p2align 6 + +.L_4X2_loop_body: + /* We clobber r8, so store it on the stack */ + movq %r8, 0(%rsp) + + /* Decode 5 symbols from each of the 4 streams (20 symbols total). */ + FOR_EACH_STREAM_WITH_INDEX(DECODE, 0) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 1) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 2) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 3) + FOR_EACH_STREAM_WITH_INDEX(DECODE, 4) + + /* Reload r8 */ + movq 0(%rsp), %r8 + + FOR_EACH_STREAM(RELOAD_BITS) + + cmp %op3, 48(%rsp) + ja .L_4X2_loop_body + jmp .L_4X2_compute_olimit + +#undef DECODE +#undef RELOAD_BITS +.L_4X2_exit: + addq $8, %rsp + /* Restore stack (oend & olimit) */ + pop %rax /* oend0 */ + pop %rax /* oend1 */ + pop %rax /* oend2 */ + pop %rax /* oend3 */ + pop %rax /* ilimit */ + pop %rax /* olimit */ + pop %rax /* arg */ + + /* Save ip / op / bits */ + movq %ip0, 0(%rax) + movq %ip1, 8(%rax) + movq %ip2, 16(%rax) + movq %ip3, 24(%rax) + movq %op0, 32(%rax) + movq %op1, 40(%rax) + movq %op2, 48(%rax) + movq %op3, 56(%rax) + movq %bits0, 64(%rax) + movq %bits1, 72(%rax) + movq %bits2, 80(%rax) + movq %bits3, 88(%rax) + + /* Restore registers */ + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rdi + pop %rsi + pop %rbp + pop %rdx + pop %rcx + pop %rbx + pop %rax + ret + +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.c b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.c new file mode 100644 index 000000000..309ec0d03 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted, ""); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ddict->dictID; +} diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.h b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.h new file mode 100644 index 000000000..c4ca8877a --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_ddict.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + + +#endif /* ZSTD_DDICT_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress.c b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress.c new file mode 100644 index 000000000..7bc271342 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress.c @@ -0,0 +1,2355 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ +#include "../common/zstd_internal.h" /* blockProperties_t */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ +#include "../common/bits.h" /* ZSTD_highbit32 */ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/************************************* + * Multiple DDicts Hashset internals * + *************************************/ + +#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ + +#define DDICT_HASHSET_TABLE_BASE_SIZE 64 +#define DDICT_HASHSET_RESIZE_FACTOR 2 + +/* Hash function to determine starting position of dict insertion within the table + * Returns an index between [0, hashSet->ddictPtrTableSize] + */ +static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) { + const U64 hash = XXH64(&dictID, sizeof(U32), 0); + /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */ + return hash & (hashSet->ddictPtrTableSize - 1); +} + +/* Adds DDict to a hashset without resizing it. + * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set. + * Returns 0 if successful, or a zstd error code if something went wrong. + */ +static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) { + const U32 dictID = ZSTD_getDictID_fromDDict(ddict); + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!"); + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + while (hashSet->ddictPtrTable[idx] != NULL) { + /* Replace existing ddict if inserting ddict with same dictID */ + if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) { + DEBUGLOG(4, "DictID already exists, replacing rather than adding"); + hashSet->ddictPtrTable[idx] = ddict; + return 0; + } + idx &= idxRangeMask; + idx++; + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + hashSet->ddictPtrTable[idx] = ddict; + hashSet->ddictPtrCount++; + return 0; +} + +/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and + * rehashes all values, allocates new table, frees old table. + * Returns 0 on success, otherwise a zstd error code. + */ +static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR; + const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem); + const ZSTD_DDict** oldTable = hashSet->ddictPtrTable; + size_t oldTableSize = hashSet->ddictPtrTableSize; + size_t i; + + DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize); + RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!"); + hashSet->ddictPtrTable = newTable; + hashSet->ddictPtrTableSize = newTableSize; + hashSet->ddictPtrCount = 0; + for (i = 0; i < oldTableSize; ++i) { + if (oldTable[i] != NULL) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), ""); + } + } + ZSTD_customFree((void*)oldTable, customMem); + DEBUGLOG(4, "Finished re-hash"); + return 0; +} + +/* Fetches a DDict with the given dictID + * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL. + */ +static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) { + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + for (;;) { + size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]); + if (currDictID == dictID || currDictID == 0) { + /* currDictID == 0 implies a NULL ddict entry */ + break; + } else { + idx &= idxRangeMask; /* Goes to start of table when we reach the end */ + idx++; + } + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + return hashSet->ddictPtrTable[idx]; +} + +/* Allocates space for and returns a ddict hash set + * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with. + * Returns NULL if allocation failed. + */ +static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { + ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); + DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; + ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); + return NULL; + } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; + return ret; +} + +/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself. + * Note: The ZSTD_DDict* within the table are NOT freed. + */ +static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + DEBUGLOG(4, "Freeing ddict hash set"); + if (hashSet && hashSet->ddictPtrTable) { + ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem); + } + if (hashSet) { + ZSTD_customFree(hashSet, customMem); + } +} + +/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set. + * Returns 0 on success, or a ZSTD error. + */ +static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) { + DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize); + if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), ""); + } + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), ""); + return 0; +} + +/*-************************************************************* +* Context management +***************************************************************/ +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) + + ZSTD_sizeof_DDict(dctx->ddictLocal) + + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->outBufferMode = ZSTD_bm_buffered; + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; + dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->staticSize = 0; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; +#endif + dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif + dctx->ddictSet = NULL; + ZSTD_DCtx_resetParameters(dctx); +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; +} + +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + dctx->customMem = customMem; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = dctx->customMem; + ZSTD_clearDict(dctx); + ZSTD_customFree(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif + if (dctx->ddictSet) { + ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); + dctx->ddictSet = NULL; + } + ZSTD_customFree(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on + * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then + * accordingly sets the ddict to be used to decompress the frame. + * + * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is. + * + * ZSTD_d_refMultipleDDicts must be enabled for this function to be called. + */ +static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) { + assert(dctx->refMultipleDDicts && dctx->ddictSet); + DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame"); + if (dctx->ddict) { + const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID); + if (frameDDict) { + DEBUGLOG(4, "DDict found!"); + ZSTD_clearDict(dctx); + dctx->dictID = dctx->fParams.dictID; + dctx->ddict = frameDDict; + dctx->dictUses = ZSTD_use_indefinitely; + } + } +} + + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + +/** ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + RETURN_ERROR(prefix_unknown, ""); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported, ""); + { size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); + return skippableSize; + } +} + +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + +/** ZSTD_findDecompressedSize() : + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; + + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; + } + /* skip to next frame */ + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + + +/** ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * If multiple DDict references are enabled, also will choose the correct DDict to use. + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) return result; /* invalid header */ + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); + + /* Reference DDict requested by frame if dctx references multiple ddicts */ + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { + ZSTD_DCtx_selectFrameDDict(dctx); + } + +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), + dictionary_wrong, ""); +#endif + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); + dctx->processedCSize += headerSize; + return 0; +} + +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} + +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize) +{ + ZSTD_frameSizeInfo frameSizeInfo; + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameSizeInfoLegacy(src, srcSize); +#endif + + if ((srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); + return frameSizeInfo; + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + size_t nbBlocks = 0; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Iterate over each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; + + if (blockProperties.lastBlock) break; + } + + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + ip += 4; + } + + frameSizeInfo.nbBlocks = nbBlocks; + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : (unsigned long long)nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; + } +} + +/** ZSTD_findFrameCompressedSize() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame + * `srcSize` must be at least as large as the frame contained + * @return : the compressed size of the frame starting at `src` */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + return frameSizeInfo.compressedSize; +} + +/** ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippeable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} + +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_frameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +/** ZSTD_insertBlock() : + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); + ZSTD_checkContinuity(dctx, blockStart, blockSize); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (srcSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memmove(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (regenSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memset(dst, b, regenSize); + return regenSize; +} + +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +{ +#if ZSTD_TRACE + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + if (dctx->ddict) { + trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict); + trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict); + trace.dictionaryIsCold = dctx->ddictIsCold; + } + trace.uncompressedSize = (size_t)uncompressedSize; + trace.compressedSize = (size_t)compressedSize; + trace.dctx = dctx; + ZSTD_trace_decompress_end(dctx->traceCtx, &trace); + } +#else + (void)dctx; + (void)uncompressedSize; + (void)compressedSize; + (void)streaming; +#endif +} + + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* const istart = (const BYTE*)(*srcPtr); + const BYTE* ip = istart; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + BYTE* oBlockEnd = oend; + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, /* frame */ 1, not_streaming); + break; + case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->validateChecksum) + XXH64_update(&dctx->xxhState, op, decodedSize); + if (decodedSize != 0) + op += decodedSize; + assert(ip != NULL); + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected, ""); + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } + ip += 4; + remainingSrcSize -= 4; + } + ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); + /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr); + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return (size_t)(op-ostart); +} + +static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) return frameSize; + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, + "legacy support is not compatible with static dctx"); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + + assert(decodedSize <= dstCapacity); + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + if (srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; /* check next frame */ + } } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); + } + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); + if (ZSTD_isError(res)) return res; + assert(res <= dstCapacity); + if (res != 0) + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); + + return (size_t)((BYTE*)dst - (BYTE*)dststart); +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + ZSTD_FALLTHROUGH; + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +/** + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return BOUNDED(1, inputSize, dctx->expected); +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + ZSTD_FALLTHROUGH; + case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + dctx->processedCSize += srcSize; + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 1, is_streaming); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_raw : + assert(srcSize <= dctx->expected); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_reserved : /* should never happen */ + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(rSize, ""); + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected, ""); + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize, /* flags */ 0); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, + dictPtr, (size_t)(dictEnd - dictPtr), + workspace, workspaceSize, /* flags */ 0); +#endif + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, + dictionary_corrupted, ""); + entropy->rep[i] = rep; + } } + + return (size_t)(dictPtr - (const BYTE*)dict); +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); +#if ZSTD_TRACE + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; +#endif + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->processedCSize = 0; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + dctx->bType = bt_reserved; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (dict && dictSize) + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted, ""); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", + dctx->ddictIsCold ? "~cold~" : "hot!"); + } + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompress frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (dict && dictSize != 0) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; + } + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); + dctx->dictUses = ZSTD_use_once; + return 0; +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); + return ZSTD_startingInputLength(zds->format); +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); + return ZSTD_startingInputLength(dctx->format); +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + DEBUGLOG(4, "ZSTD_resetDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); + return ZSTD_startingInputLength(dctx->format); +} + + +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) { + if (dctx->ddictSet == NULL) { + dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem); + if (!dctx->ddictSet) { + RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!"); + } + } + assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */ + FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), ""); + } + } + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; + return bounds; + case ZSTD_d_refMultipleDDicts: + bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; + bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; + return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ +} + +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + case ZSTD_d_refMultipleDDicts: + *value = (int)dctx->refMultipleDDicts; + return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + switch(dParam) { + case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; + return 0; + case ZSTD_d_refMultipleDDicts: + CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value); + if (dctx->staticSize != 0) { + RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!"); + } + dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; + return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + ZSTD_DCtx_resetParameters(dctx); + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + size_t const blockSize = (size_t) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + /* space is needed to store the litbuffer after the output of a given block without stomping the extDict of a previous run, as well as to cover both windows against wildcopy*/ + unsigned long long const neededRBSize = windowSize + blockSize + ZSTD_BLOCKSIZE_MAX + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge, ""); + return minRBSize; +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge, ""); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + + +/* ***** Decompression ***** */ + +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_bm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); +} + +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (zds->outBufferMode == ZSTD_bm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(zds->outBufferMode == ZSTD_bm_stable); + } + return 0; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; + const char* ip = istart; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + DEBUGLOG(5, "stage zdss_init => transparent reset "); + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + zds->legacyVersion = 0; +#endif + zds->hostageByte = 0; + zds->expectedOutBuffer = *output; + ZSTD_FALLTHROUGH; + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) { + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; + return hint; + } } +#endif + { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + if (zds->refMultipleDDicts && zds->ddictSet) { + ZSTD_DCtx_selectFrameDDict(zds); + } + if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); + const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; + size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; + DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, + zds->previousLegacyVersion, legacyVersion, + dict, dictSize), ""); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ + return hint; + } } +#endif + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend-ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize(istart, (size_t)(iend-istart)); + if (cSize <= (size_t)(iend-istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + assert(istart != NULL); + ip = istart + cSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (zds->outBufferMode == ZSTD_bm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); + + if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >>10), + (U32)(zds->maxWindowSize >> 10) ); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge, ""); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_min(zds->fParams.windowSize, zds->fParams.frameContentSize) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_customFree(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } + zds->streamStage = zdss_read; + ZSTD_FALLTHROUGH; + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); + ip += neededInSize; + /* Function modifies the stage so we must break */ + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + ZSTD_FALLTHROUGH; + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend-ip)); + } else { + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); + } + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } + case zdss_flush: + { + size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); + + op = op ? op + flushedSize : op; + + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + + if ((ip==istart) && (op==ostart)) { /* no forward progress */ + zds->noForwardProgress ++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} + +size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.c b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.c new file mode 100644 index 000000000..09896a931 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.c @@ -0,0 +1,2192 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/compiler.h" /* prefetch */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" +#include "../common/bits.h" /* ZSTD_highbit32 */ + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); + + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); + return cSize; + } +} + +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + if (streaming == not_streaming && dstCapacity > ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) + { + /* room for litbuffer to fit without read faulting */ + dctx->litBuffer = (BYTE*)dst + ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } + else if (litSize > ZSTD_LITBUFFEREXTRASIZE) + { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } + else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + } + else + { + /* fits entirely within litExtraBuffer, so no split is necessary */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } +} + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity, const streaming_operation streaming); +/*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) +{ + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); + ZSTD_FALLTHROUGH; + + case set_compressed: + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } else { + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); + } + } + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + } + + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + RETURN_ERROR(corruption_detected, "impossible"); + } + } +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what empirically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].baseValue = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold)) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u=0; u max, corruption_detected, ""); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); + *DTablePtr = DTableSpace; + return headerSize; + } + default : + assert(0); + RETURN_ERROR(GENERIC, "impossible"); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); + + /* SeqHead */ + nbSeq = *ip++; + if (!nbSeq) { + *nbSeqPtr=0; + RETURN_ERROR_IF(srcSize != 1, srcSize_wrong, ""); + return 1; + } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; + } else { + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + /* FSE table descriptors */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; +} seqState_t; + +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op += oend_w - op; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, BYTE const* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) +{ + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offsets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +{ + seq_t seq; + /* + * ZSTD_seqSymbol is a structure with a total of 64 bits wide. So it can be + * loaded in one operation and extracted its fields by simply shifting or + * bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. + */ +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + } + + return seq; +} + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION +MEM_STATIC int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} + +MEM_STATIC void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ +#if DEBUGLEVEL >= 1 + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_BLOCKSIZE_MAX); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif +} +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer"); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + + /* decompress without overrunning litPtr begins */ + { + seq_t sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif +#endif + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for (; litPtr + sequence.litLength <= dctx->litBufferEnd; ) { + size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + } + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (--nbSeq) + BIT_reloadDStream(&(seqState.DStream)); + } + } + } + + if (nbSeq > 0) /* there is remaining lit from extra buffer */ + { + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for (; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + { + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ostart + maxDstSize : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; ; ) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + if (UNLIKELY(!--nbSeq)) + break; + BIT_reloadDStream(&(seqState.DStream)); + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + RETURN_ERROR_IF(BIT_reloadDStream(&seqState.DStream) < BIT_DStream_completed, corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE size_t +ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + const BYTE* const match = matchBase + prefetchPos - sequence.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + (void)frame; + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 8 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS STORED_SEQS + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + + dctx->fseEntropy = 1; + { int i; for (i=0; ientropy.rep[i]; } + assert(dst != NULL); + assert(iend >= ip); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) + { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + } + RETURN_ERROR_IF(seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) + { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { + size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + if (frame) ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) /* first deplete literal buffer in dst, then copy litExtraBuffer */ + { + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset, + const int frame) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset, frame); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} + +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + } + + return info; +} + +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } +} + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame, const streaming_operation streaming) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX. + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX)); + size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); + + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } + } + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); +#endif + } +} + + +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) +{ + if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0, not_streaming); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.h b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.h new file mode 100644 index 000000000..9d1318882 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_block.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* DCtx, and some public functions */ +#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame, const streaming_operation streaming); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); + +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_internal.h b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_internal.h new file mode 100644 index 000000000..c2ec5d9fb --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/decompress/zstd_decompress_internal.h @@ -0,0 +1,238 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/mem.h" /* BYTE, U16, U32 */ +#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; + +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; +#if DYNAMIC_BMI2 != 0 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; +#endif + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; +#endif +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 != 0 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/External/Zstd/zstd-1.5.5/lib/zdict.h b/External/Zstd/zstd-1.5.5/lib/zdict.h new file mode 100644 index 000000000..2268f948a --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/zdict.h @@ -0,0 +1,474 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_ZDICT_H +#define ZSTD_ZDICT_H + +/*====== Dependencies ======*/ +#include /* size_t */ + + +/* ===== ZDICTLIB_API : control library symbols visibility ===== */ +#ifndef ZDICTLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZDICTLIB_VISIBLE +# endif +#endif + +#ifndef ZDICTLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZDICTLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZDICTLIB_API ZDICTLIB_VISIBLE +#endif + +/******************************************************************************* + * Zstd dictionary builder + * + * FAQ + * === + * Why should I use a dictionary? + * ------------------------------ + * + * Zstd can use dictionaries to improve compression ratio of small data. + * Traditionally small files don't compress well because there is very little + * repetition in a single sample, since it is small. But, if you are compressing + * many similar files, like a bunch of JSON records that share the same + * structure, you can train a dictionary on ahead of time on some samples of + * these files. Then, zstd can use the dictionary to find repetitions that are + * present across samples. This can vastly improve compression ratio. + * + * When is a dictionary useful? + * ---------------------------- + * + * Dictionaries are useful when compressing many small files that are similar. + * The larger a file is, the less benefit a dictionary will have. Generally, + * we don't expect dictionary compression to be effective past 100KB. And the + * smaller a file is, the more we would expect the dictionary to help. + * + * How do I use a dictionary? + * -------------------------- + * + * Simply pass the dictionary to the zstd compressor with + * `ZSTD_CCtx_loadDictionary()`. The same dictionary must then be passed to + * the decompressor, using `ZSTD_DCtx_loadDictionary()`. There are other + * more advanced functions that allow selecting some options, see zstd.h for + * complete documentation. + * + * What is a zstd dictionary? + * -------------------------- + * + * A zstd dictionary has two pieces: Its header, and its content. The header + * contains a magic number, the dictionary ID, and entropy tables. These + * entropy tables allow zstd to save on header costs in the compressed file, + * which really matters for small data. The content is just bytes, which are + * repeated content that is common across many samples. + * + * What is a raw content dictionary? + * --------------------------------- + * + * A raw content dictionary is just bytes. It doesn't have a zstd dictionary + * header, a dictionary ID, or entropy tables. Any buffer is a valid raw + * content dictionary. + * + * How do I train a dictionary? + * ---------------------------- + * + * Gather samples from your use case. These samples should be similar to each + * other. If you have several use cases, you could try to train one dictionary + * per use case. + * + * Pass those samples to `ZDICT_trainFromBuffer()` and that will train your + * dictionary. There are a few advanced versions of this function, but this + * is a great starting point. If you want to further tune your dictionary + * you could try `ZDICT_optimizeTrainFromBuffer_cover()`. If that is too slow + * you can try `ZDICT_optimizeTrainFromBuffer_fastCover()`. + * + * If the dictionary training function fails, that is likely because you + * either passed too few samples, or a dictionary would not be effective + * for your data. Look at the messages that the dictionary trainer printed, + * if it doesn't say too few samples, then a dictionary would not be effective. + * + * How large should my dictionary be? + * ---------------------------------- + * + * A reasonable dictionary size, the `dictBufferCapacity`, is about 100KB. + * The zstd CLI defaults to a 110KB dictionary. You likely don't need a + * dictionary larger than that. But, most use cases can get away with a + * smaller dictionary. The advanced dictionary builders can automatically + * shrink the dictionary for you, and select the smallest size that doesn't + * hurt compression ratio too much. See the `shrinkDict` parameter. + * A smaller dictionary can save memory, and potentially speed up + * compression. + * + * How many samples should I provide to the dictionary builder? + * ------------------------------------------------------------ + * + * We generally recommend passing ~100x the size of the dictionary + * in samples. A few thousand should suffice. Having too few samples + * can hurt the dictionaries effectiveness. Having more samples will + * only improve the dictionaries effectiveness. But having too many + * samples can slow down the dictionary builder. + * + * How do I determine if a dictionary will be effective? + * ----------------------------------------------------- + * + * Simply train a dictionary and try it out. You can use zstd's built in + * benchmarking tool to test the dictionary effectiveness. + * + * # Benchmark levels 1-3 without a dictionary + * zstd -b1e3 -r /path/to/my/files + * # Benchmark levels 1-3 with a dictionary + * zstd -b1e3 -r /path/to/my/files -D /path/to/my/dictionary + * + * When should I retrain a dictionary? + * ----------------------------------- + * + * You should retrain a dictionary when its effectiveness drops. Dictionary + * effectiveness drops as the data you are compressing changes. Generally, we do + * expect dictionaries to "decay" over time, as your data changes, but the rate + * at which they decay depends on your use case. Internally, we regularly + * retrain dictionaries, and if the new dictionary performs significantly + * better than the old dictionary, we will ship the new dictionary. + * + * I have a raw content dictionary, how do I turn it into a zstd dictionary? + * ------------------------------------------------------------------------- + * + * If you have a raw content dictionary, e.g. by manually constructing it, or + * using a third-party dictionary builder, you can turn it into a zstd + * dictionary by using `ZDICT_finalizeDictionary()`. You'll also have to + * provide some samples of the data. It will add the zstd header to the + * raw content, which contains a dictionary ID and entropy tables, which + * will improve compression ratio, and allow zstd to write the dictionary ID + * into the frame, if you so choose. + * + * Do I have to use zstd's dictionary builder? + * ------------------------------------------- + * + * No! You can construct dictionary content however you please, it is just + * bytes. It will always be valid as a raw content dictionary. If you want + * a zstd dictionary, which can improve compression ratio, use + * `ZDICT_finalizeDictionary()`. + * + * What is the attack surface of a zstd dictionary? + * ------------------------------------------------ + * + * Zstd is heavily fuzz tested, including loading fuzzed dictionaries, so + * zstd should never crash, or access out-of-bounds memory no matter what + * the dictionary is. However, if an attacker can control the dictionary + * during decompression, they can cause zstd to generate arbitrary bytes, + * just like if they controlled the compressed data. + * + ******************************************************************************/ + + +/*! ZDICT_trainFromBuffer(): + * Train a dictionary from an array of samples. + * Redirect towards ZDICT_optimizeTrainFromBuffer_fastCover() single-threaded, with d=8, steps=4, + * f=20, and accel=1. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * Note: Dictionary training will fail if there are not enough samples to construct a + * dictionary, or if most of the samples are too small (< 8 bytes being the lower limit). + * If dictionary training fails, you should use zstd without a dictionary, as the dictionary + * would've been ineffective anyways. If you believe your samples would benefit from a dictionary + * please open an issue with details, and we can look into it. + * Note: ZDICT_trainFromBuffer()'s memory usage is about 6 MB. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples); + +typedef struct { + int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value) + * NOTE: The zstd format reserves some dictionary IDs for future use. + * You may use them in private settings, but be warned that they + * may be used by zstd in a public dictionary registry in the future. + * These dictionary IDs are: + * - low range : <= 32767 + * - high range : >= (2^31) + */ +} ZDICT_params_t; + +/*! ZDICT_finalizeDictionary(): + * Given a custom content as a basis for dictionary, and a set of samples, + * finalize dictionary by adding headers and statistics according to the zstd + * dictionary format. + * + * Samples must be stored concatenated in a flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each + * sample in order. The samples are used to construct the statistics, so they + * should be representative of what you will compress with this dictionary. + * + * The compression level can be set in `parameters`. You should pass the + * compression level you expect to use in production. The statistics for each + * compression level differ, so tuning the dictionary for the compression level + * can help quite a bit. + * + * You can set an explicit dictionary ID in `parameters`, or allow us to pick + * a random dictionary ID for you, but we can't guarantee no collisions. + * + * The dstDictBuffer and the dictContent may overlap, and the content will be + * appended to the end of the header. If the header + the content doesn't fit in + * maxDictSize the beginning of the content is truncated to make room, since it + * is presumed that the most profitable content is at the end of the dictionary, + * since that is the cheapest to reference. + * + * `maxDictSize` must be >= max(dictContentSize, ZSTD_DICTSIZE_MIN). + * + * @return: size of dictionary stored into `dstDictBuffer` (<= `maxDictSize`), + * or an error code, which can be tested by ZDICT_isError(). + * Note: ZDICT_finalizeDictionary() will push notifications into stderr if + * instructed to, using notificationLevel>0. + * NOTE: This function currently may fail in several edge cases including: + * * Not enough samples + * * Samples are uncompressible + * * Samples are all exactly the same + */ +ZDICTLIB_API size_t ZDICT_finalizeDictionary(void* dstDictBuffer, size_t maxDictSize, + const void* dictContent, size_t dictContentSize, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + + +/*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize); /* returns dict header size; returns a ZSTD error code on failure */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); + +#endif /* ZSTD_ZDICT_H */ + +#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC) +#define ZSTD_ZDICT_H_STATIC + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZDICTLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE +# else +# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE +# endif +#endif + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +#define ZDICT_DICTSIZE_MIN 256 +/* Deprecated: Remove in v1.6.0 */ +#define ZDICT_CONTENTSIZE_MIN 128 + +/*! ZDICT_cover_params_t: + * k and d are the only required parameters. + * For others, value 0 means default. + */ +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (1.0), 1.0 when all samples are used for both training and testing */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + ZDICT_params_t zParams; +} ZDICT_cover_params_t; + +typedef struct { + unsigned k; /* Segment size : constraint: 0 < k : Reasonable range [16, 2048+] */ + unsigned d; /* dmer size : constraint: 0 < d <= k : Reasonable range [6, 16] */ + unsigned f; /* log of size of frequency array : constraint: 0 < f <= 31 : 1 means default(20)*/ + unsigned steps; /* Number of steps : Only used for optimization : 0 means default (40) : Higher means more parameters checked */ + unsigned nbThreads; /* Number of threads : constraint: 0 < nbThreads : 1 means single-threaded : Only used for optimization : Ignored if ZSTD_MULTITHREAD is not defined */ + double splitPoint; /* Percentage of samples used for training: Only used for optimization : the first nbSamples * splitPoint samples will be used to training, the last nbSamples * (1 - splitPoint) samples will be used for testing, 0 means default (0.75), 1.0 when all samples are used for both training and testing */ + unsigned accel; /* Acceleration level: constraint: 0 < accel <= 10, higher means faster and less accurate, 0 means default(1) */ + unsigned shrinkDict; /* Train dictionaries to shrink in size starting from the minimum size and selects the smallest dictionary that is shrinkDictMaxRegression% worse than the largest dictionary. 0 means no shrinking and 1 means shrinking */ + unsigned shrinkDictMaxRegression; /* Sets shrinkDictMaxRegression so that a smaller dictionary can be at worse shrinkDictMaxRegression% worse than the max dict size dictionary. */ + + ZDICT_params_t zParams; +} ZDICT_fastCover_params_t; + +/*! ZDICT_trainFromBuffer_cover(): + * Train a dictionary from an array of samples using the COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_cover() requires about 9 bytes of memory for each input byte. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( + void *dictBuffer, size_t dictBufferCapacity, + const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_cover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations and picks the best parameters. + * `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * + * All of the parameters d, k, steps are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. + */ +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_cover_params_t* parameters); + +/*! ZDICT_trainFromBuffer_fastCover(): + * Train a dictionary from an array of samples using a modified version of COVER algorithm. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * d and k are required. + * All other parameters are optional, will use default values if not provided + * The resulting dictionary will be saved into `dictBuffer`. + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_trainFromBuffer_fastCover() requires 6 * 2^f bytes of memory. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, + size_t dictBufferCapacity, const void *samplesBuffer, + const size_t *samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t parameters); + +/*! ZDICT_optimizeTrainFromBuffer_fastCover(): + * The same requirements as above hold for all the parameters except `parameters`. + * This function tries many parameter combinations (specifically, k and d combinations) + * and picks the best parameters. `*parameters` is filled with the best parameters found, + * dictionary constructed with those parameters is stored in `dictBuffer`. + * All of the parameters d, k, steps, f, and accel are optional. + * If d is non-zero then we don't check multiple values of d, otherwise we check d = {6, 8}. + * if steps is zero it defaults to its default value. + * If k is non-zero then we don't check multiple values of k, otherwise we check steps values in [50, 2000]. + * If f is zero, default value of 20 is used. + * If accel is zero, default value of 1 is used. + * + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * On success `*parameters` contains the parameters selected. + * See ZDICT_trainFromBuffer() for details on failure modes. + * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. + */ +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, + size_t dictBufferCapacity, const void* samplesBuffer, + const size_t* samplesSizes, unsigned nbSamples, + ZDICT_fastCover_params_t* parameters); + +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ + ZDICT_params_t zParams; +} ZDICT_legacy_params_t; + +/*! ZDICT_trainFromBuffer_legacy(): + * Train a dictionary from an array of samples. + * Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + * supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + * The resulting dictionary will be saved into `dictBuffer`. + * `parameters` is optional and can be provided with values set to 0 to mean "default". + * @return: size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + * or an error code, which can be tested with ZDICT_isError(). + * See ZDICT_trainFromBuffer() for details on failure modes. + * Tips: In general, a reasonable dictionary has a size of ~ 100 KB. + * It's possible to select smaller or larger size, just by specifying `dictBufferCapacity`. + * In general, it's recommended to provide a few thousands samples, though this can vary a lot. + * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. + * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. + */ +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy( + void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_legacy_params_t parameters); + + +/* Deprecation warnings */ +/* It is generally possible to disable deprecation warnings from compiler, + for example with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual. + Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ +#ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS +# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */ +#else +# define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] +# elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (ZDICT_GCC_VERSION >= 301) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZDICT_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") +# define ZDICT_DEPRECATED(message) +# endif +#endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ + +ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +ZDICTLIB_STATIC_API +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + +#endif /* ZSTD_ZDICT_H_STATIC */ + +#if defined (__cplusplus) +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/zstd.h b/External/Zstd/zstd-1.5.5/lib/zstd.h new file mode 100644 index 000000000..e5c3f8b68 --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/zstd.h @@ -0,0 +1,3020 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependencies ======*/ +#include /* INT_MAX */ +#include /* size_t */ + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 5 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +/* ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()` or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a context just once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to behave similarly to `ZSTD_compress()`, + * this function compresses at requested compression level, + * __ignoring any other parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and re-use it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters. + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove from experimental API entry points which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_useBlockSplitter + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will re-use the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-used multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* But if `output.pos == output.size`, there might be some data left within internal buffers., +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder finished and flushed all remaining buffers. + * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers, + * call ZSTD_decompressStream() again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + */ +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is re-used, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final + * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable + * or ZSTD_ps_disable allow for a force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_paramSwitch_e; + +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_frameHeader; + +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + +/*! ZSTD_generateSequences() : + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2(). + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters + * @return : number of sequences generated + */ + +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences( ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsSize, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * for any compression level up to selected one. + * Note : Unlike ZSTD_estimateCStreamSize*(), this estimate + * does not include space for a window buffer. + * Therefore, the estimation is only guaranteed for single-shot compressions, not streaming. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * When srcSize can be bound by a known and rather "small" value, + * this fact can be used to provide a tighter estimation + * because the CCtx compression context will need less memory. + * This tighter estimation can be provided by more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * + * Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one. + * It will also consider src size to be arbitrarily "large", which is worst case. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_DStream memory budget depends on window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + * Note 2 : only single-threaded compression is supported. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t windowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* Tries to fit compressed block size to be around targetCBlockSize. + * No target when targetCBlockSize == 0. + * There is no guarantee on compressed block size (default:0) */ +#define ZSTD_c_targetCBlockSize ZSTD_c_experimentalParam6 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if conditions are not respected. + * + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to rely on user provided buffer instead. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_useBlockSplitter + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + * + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_searchForExternalRepcodes + * This parameter affects how zstd parses external sequences, such as sequences + * provided through the compressSequences() API or from an external block-level + * sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets in + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level. + * + * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future. + */ +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * re-use decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t ZSTD_sequenceProducer_F ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F* sequenceProducer +); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ + +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ + +/*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff --git a/External/Zstd/zstd-1.5.5/lib/zstd_errors.h b/External/Zstd/zstd-1.5.5/lib/zstd_errors.h new file mode 100644 index 000000000..dc75eeeba --- /dev/null +++ b/External/Zstd/zstd-1.5.5/lib/zstd_errors.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stabilityCondition_notRespected = 50, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ From 1b3935069d29b965a9d1d6798d9cf651319dbe87 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 12:26:25 +0930 Subject: [PATCH 078/129] Fix bad merge --- Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp | 9 --------- Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp | 1 - Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h | 2 -- 3 files changed, 12 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index fd2f81d49..aacc2bfba 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -613,15 +613,6 @@ void Server::FindNeedyClients() } PROFILE_FUNCTION; - - // determine job availability - int32_t availableJobs = (int32_t)WorkerThreadRemote::GetNumCPUsToUse(); - if ( availableJobs == 0 ) - { - return; - } - ++availableJobs; // over request to parallelize building/network transfers - // determine job availability int32_t availableJobs = (int32_t)WorkerThreadRemote::GetNumCPUsToUse(); diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp index daf6b604b..88a6473d7 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp @@ -47,7 +47,6 @@ void WorkerThread::Init( ThreadPool * pool ) WorkerThread::~WorkerThread() { ASSERT( m_Exited.Load() ); - m_Thread.Join(); } // InitTmpDir diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h index 3149bed3f..a146fa5b2 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h @@ -8,7 +8,6 @@ #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" #include "Core/Process/Semaphore.h" -#include "Core/Process/Thread.h" #include "Core/Strings/AStackString.h" #include "Core/Strings/AString.h" @@ -51,7 +50,6 @@ class WorkerThread virtual void Main(); // signal to exit thread - Thread m_Thread; Atomic m_ShouldExit; Atomic m_Exited; uint16_t m_ThreadIndex; From 6d789bb4f05896528502e3ae030a332a2ac423fa Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 12:29:05 +0930 Subject: [PATCH 079/129] Fix bad merge --- Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp | 9 --------- Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp | 1 - Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h | 2 -- 3 files changed, 12 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index fd2f81d49..aacc2bfba 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -613,15 +613,6 @@ void Server::FindNeedyClients() } PROFILE_FUNCTION; - - // determine job availability - int32_t availableJobs = (int32_t)WorkerThreadRemote::GetNumCPUsToUse(); - if ( availableJobs == 0 ) - { - return; - } - ++availableJobs; // over request to parallelize building/network transfers - // determine job availability int32_t availableJobs = (int32_t)WorkerThreadRemote::GetNumCPUsToUse(); diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp index daf6b604b..88a6473d7 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.cpp @@ -47,7 +47,6 @@ void WorkerThread::Init( ThreadPool * pool ) WorkerThread::~WorkerThread() { ASSERT( m_Exited.Load() ); - m_Thread.Join(); } // InitTmpDir diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h index 3149bed3f..a146fa5b2 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h @@ -8,7 +8,6 @@ #include "Core/Process/Atomic.h" #include "Core/Process/Mutex.h" #include "Core/Process/Semaphore.h" -#include "Core/Process/Thread.h" #include "Core/Strings/AStackString.h" #include "Core/Strings/AString.h" @@ -51,7 +50,6 @@ class WorkerThread virtual void Main(); // signal to exit thread - Thread m_Thread; Atomic m_ShouldExit; Atomic m_Exited; uint16_t m_ThreadIndex; From 0b26cdb5f6c6de478efb34ba11a06dbb03804377 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 13:24:37 +0930 Subject: [PATCH 080/129] Fix v1.10 download links --- Code/Tools/FBuild/Documentation/docs/download.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/download.html b/Code/Tools/FBuild/Documentation/docs/download.html index b871b67a9..6e11eae7d 100644 --- a/Code/Tools/FBuild/Documentation/docs/download.html +++ b/Code/Tools/FBuild/Documentation/docs/download.html @@ -88,10 +88,10 @@

Editor Integration

VersionWindowsOS XLinuxSourceMarkup
v1.10x64x64x64Zip | + GitHubNotePad++Visual StudioBash completion
v1.09 x64 x64 Generate JSON compilation database for the specified targets.
-config [path]-config <path> Explicitly specify the config file to use.
-continueafterdbmove Allow build to continue after a DB move.
-dbfile <path>Explicitly specify the dependency database file to use.
-debug [Windows Only] Allow attaching a debugger immediately on startup.
- - - - + + + From 56c3284ab0fe58e9a2d95af8ea59bf7e369735fe Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 13:25:00 +0930 Subject: [PATCH 081/129] Remove private support email link --- Code/Tools/FBuild/Documentation/docs/contact.html | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/contact.html b/Code/Tools/FBuild/Documentation/docs/contact.html index 097c69d04..060e02b9c 100644 --- a/Code/Tools/FBuild/Documentation/docs/contact.html +++ b/Code/Tools/FBuild/Documentation/docs/contact.html @@ -61,21 +61,6 @@

Contact and Support

- -
- Private Support -
-
-

If the documentation or GitHub doesn't help, or your question can't be asked publicly, you can send an email for private support. -

    -
    • -
-Please note that responses may be significantly delayed as support load is highly variable! -

-
- - -
© 2012-2023 Franta Fulin
\ No newline at end of file From 94a32a38bac153678d89cb29ed2757d880d48f5f Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 13:33:05 +0930 Subject: [PATCH 082/129] Update 'dev' branch to compile with v1.10 --- .github/actions/DownloadFBuild/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/DownloadFBuild/action.yml b/.github/actions/DownloadFBuild/action.yml index dbd0d17be..f740d75df 100644 --- a/.github/actions/DownloadFBuild/action.yml +++ b/.github/actions/DownloadFBuild/action.yml @@ -5,7 +5,7 @@ inputs: version: description: Version to download required: false - default: 1.09 + default: 1.10 runs: using: composite From f43909c5e78603525fcd58da30befaf7989bb7d9 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 13:37:05 +0930 Subject: [PATCH 083/129] Quote version string to prevent stripping trailing zeros --- .github/actions/DownloadFBuild/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/DownloadFBuild/action.yml b/.github/actions/DownloadFBuild/action.yml index f740d75df..2e1d9d7e1 100644 --- a/.github/actions/DownloadFBuild/action.yml +++ b/.github/actions/DownloadFBuild/action.yml @@ -5,7 +5,7 @@ inputs: version: description: Version to download required: false - default: 1.10 + default: "1.10" runs: using: composite From 49ffe0bb9e625d45cd79dea67c0de439a7a9ddb9 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 13:38:35 +0930 Subject: [PATCH 084/129] Switch OSX CI to use universal binary version --- .github/actions/DownloadFBuild/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/DownloadFBuild/action.yml b/.github/actions/DownloadFBuild/action.yml index 2e1d9d7e1..5611b2c6e 100644 --- a/.github/actions/DownloadFBuild/action.yml +++ b/.github/actions/DownloadFBuild/action.yml @@ -15,7 +15,7 @@ runs: run: | case "${{ runner.os }}" in Linux) FN="FASTBuild-Linux-x64-v${{ inputs.version }}.zip" FBUILD=fbuild ;; - macOS) FN="FASTBuild-OSX-x64-v${{ inputs.version }}.zip" FBUILD=FBuild ;; + macOS) FN="FASTBuild-OSX-x64+ARM-v${{ inputs.version }}.zip" FBUILD=FBuild ;; Windows) FN="FASTBuild-Windows-x64-v${{ inputs.version }}.zip" FBUILD=FBuild.exe ;; esac curl -fL "https://fastbuild.org/downloads/v${{ inputs.version }}/${FN}" -o "${FN}" From 1f3058ef68554ae87c288eb378a750481cd5e9e8 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 19 Aug 2023 13:42:39 +0930 Subject: [PATCH 085/129] Update v1.10 link text to reflection it's a Universal Binary --- Code/Tools/FBuild/Documentation/docs/download.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/download.html b/Code/Tools/FBuild/Documentation/docs/download.html index 6e11eae7d..93bbc37c0 100644 --- a/Code/Tools/FBuild/Documentation/docs/download.html +++ b/Code/Tools/FBuild/Documentation/docs/download.html @@ -41,7 +41,7 @@

Download

- + @@ -89,7 +89,7 @@

Editor Integration

- + From 99ff043e809aae0b8d81346486ed7cd72730f612 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 20 Aug 2023 18:21:11 +0930 Subject: [PATCH 086/129] [Improvement] -distverbose emits worker version info for v1.12 workers onwards - new Workers send a ConnectionAck message to clients on connection, which included Worker version info - when new Clients receive this message, they store info and can emit version info to the log when -distverbose is enabled --- .../FBuild/FBuildCore/Protocol/Client.cpp | 29 +++++++++++++++++++ .../Tools/FBuild/FBuildCore/Protocol/Client.h | 4 +++ .../FBuild/FBuildCore/Protocol/Protocol.cpp | 15 ++++++++++ .../FBuild/FBuildCore/Protocol/Protocol.h | 22 +++++++++++++- .../FBuild/FBuildCore/Protocol/Server.cpp | 8 +++++ 5 files changed, 77 insertions(+), 1 deletion(-) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp index 85d9a2681..f17490f3c 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp @@ -399,6 +399,12 @@ void Client::SendMessageInternal( const ConnectionInfo * connection, const Proto Process( connection, msg ); break; } + case Protocol::MSG_CONNECTION_ACK: + { + const Protocol::MsgConnectionAck * msg = static_cast< const Protocol::MsgConnectionAck * >( imsg ); + Process( connection, msg ); + break; + } default: { // unknown message type @@ -516,6 +522,29 @@ void Client::Process( const ConnectionInfo * connection, const Protocol::MsgJobR ProcessJobResultCommon( connection, compressed, payload, payloadSize ); } +// Process( MsgConnectionAck ) +//------------------------------------------------------------------------------ +void Client::Process( const ConnectionInfo * connection, const Protocol::MsgConnectionAck * msg ) +{ + PROFILE_SECTION( "MsgConnectionAck" ); + + ServerState * ss = (ServerState *)connection->GetUserData(); + ASSERT( ss ); + + // The connection would be dropped and we would not get an ack if the major + // protocol version is mismatched. + ASSERT( msg->GetProtocolVersionMajor() == Protocol::PROTOCOL_VERSION_MAJOR ); + + // Take note of additional server info + ss->m_WorkerVersion.Store( msg->GetWorkerVersion() ); + ss->m_ProtocolVersionMinor.Store( msg->GetProtocolVersionMinor() ); + DIST_INFO( " - Worker %s is v%u.%u (protocol v%u.%u)\n", ss->m_RemoteName.Get(), + (ss->m_WorkerVersion.Load() / 100U), + (ss->m_WorkerVersion.Load() % 100U), + Protocol::PROTOCOL_VERSION_MAJOR, + ss->m_ProtocolVersionMinor.Load() ); +} + // ProcessJobResultCommon //------------------------------------------------------------------------------ void Client::ProcessJobResultCommon( const ConnectionInfo * connection, bool isCompressed, const void * payload, size_t payloadSize ) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.h b/Code/Tools/FBuild/FBuildCore/Protocol/Client.h index 8b3462475..b8b52ee28 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.h @@ -20,6 +20,7 @@ class MultiBuffer; namespace Protocol { class IMessage; + class MsgConnectionAck; class MsgJobResult; class MsgJobResultCompressed; class MsgRequestJob; @@ -49,6 +50,7 @@ class Client : public TCPConnectionPool void Process( const ConnectionInfo * connection, const Protocol::MsgJobResultCompressed * msg, const void * payload, size_t payloadSize ); void Process( const ConnectionInfo * connection, const Protocol::MsgRequestManifest * msg ); void Process( const ConnectionInfo * connection, const Protocol::MsgRequestFile * msg ); + void Process( const ConnectionInfo * connection, const Protocol::MsgConnectionAck * msg ); void ProcessJobResultCommon( const ConnectionInfo * connection, bool isCompressed, const void * payload, size_t payloadSize ); @@ -80,6 +82,8 @@ class Client : public TCPConnectionPool const ConnectionInfo * m_Connection; AString m_RemoteName; + Atomic m_WorkerVersion; + Atomic m_ProtocolVersionMinor; Mutex m_Mutex; const Protocol::IMessage * m_CurrentMessage; diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Protocol.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Protocol.cpp index 371490d27..87aa73662 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Protocol.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Protocol.cpp @@ -5,6 +5,10 @@ //------------------------------------------------------------------------------ #include "Protocol.h" +// Tools/FBuild +#include "Tools/FBuild/FBuildCore/FBuildVersion.h" + +// Core #include "Core/Env/Env.h" #include "Core/FileIO/ConstMemoryStream.h" #include "Core/FileIO/MemoryStream.h" @@ -36,6 +40,7 @@ "RequestFile", "File", "JobResultCompressed", + "ConnectionAck", }; static_assert( ( sizeof( msgNames ) / sizeof(const char *) ) == Protocol::NUM_MESSAGES, "msgNames item count doesn't match NUM_MESSAGES" ); @@ -115,6 +120,16 @@ Protocol::MsgConnection::MsgConnection( uint32_t numJobsAvailable ) } } +// MsgConnectionAck +//------------------------------------------------------------------------------ +Protocol::MsgConnectionAck::MsgConnectionAck() + : Protocol::IMessage( Protocol::MSG_CONNECTION_ACK, sizeof( MsgConnectionAck ), false ) + , m_WorkerVersion( static_cast( FBUILD_VERSION ) ) + , m_ProtocolVersionMajor( PROTOCOL_VERSION_MAJOR ) + , m_ProtocolVersionMinor( PROTOCOL_VERSION_MINOR ) +{ +} + // MsgStatus //------------------------------------------------------------------------------ Protocol::MsgStatus::MsgStatus( uint32_t numJobsAvailable ) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Protocol.h b/Code/Tools/FBuild/FBuildCore/Protocol/Protocol.h index 3784326d7..8aea7e45c 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Protocol.h +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Protocol.h @@ -32,7 +32,7 @@ namespace Protocol // Protocol Version enum : uint32_t { PROTOCOL_VERSION_MAJOR = 22 }; // Changes here make workers incompatible - enum : uint8_t { PROTOCOL_VERSION_MINOR = 2 }; // Changes must be forwards and backwards compatible + enum : uint8_t { PROTOCOL_VERSION_MINOR = 3 }; // Changes must be forwards and backwards compatible enum { PROTOCOL_TEST_PORT = PROTOCOL_PORT + 1 }; // Different port for use by tests @@ -55,8 +55,12 @@ namespace Protocol MSG_REQUEST_FILE = 9, // Server -> Client : Ask client for a file MSG_FILE = 10,// Server <- Client : Send a requested file + // v22.1 or later MSG_JOB_RESULT_COMPRESSED = 11, // Server -> Client : Return completed job (compressed) + // v22.3 or later + MSG_CONNECTION_ACK = 12,// Server -> Client : Handshake ack + NUM_MESSAGES // leave last }; }; @@ -114,6 +118,22 @@ namespace Protocol }; static_assert( sizeof( MsgConnection ) == sizeof( IMessage ) + 76, "MsgConnection message has incorrect size" ); + // MsgConnectionAck + //------------------------------------------------------------------------------ + class MsgConnectionAck : public IMessage + { + public: + MsgConnectionAck(); + + uint16_t GetWorkerVersion() const { return m_WorkerVersion; } + uint8_t GetProtocolVersionMajor() const { return m_ProtocolVersionMajor; } + uint8_t GetProtocolVersionMinor() const { return m_ProtocolVersionMinor; } + private: + uint16_t m_WorkerVersion; + uint8_t m_ProtocolVersionMajor; + uint8_t m_ProtocolVersionMinor; + }; + // MsgStatus //------------------------------------------------------------------------------ class MsgStatus : public IMessage diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index aacc2bfba..d587fd95b 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -293,6 +293,14 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgConn cs->m_NumJobsAvailable.Store( msg->GetNumJobsAvailable() ); cs->m_ProtocolVersionMinor = msg->GetProtocolVersionMinor(); cs->m_HostName = msg->GetHostName(); + + // If Client is new enough, send an ack message + if ( msg->GetProtocolVersionMinor() >= 3 ) + { + // Send Ack to client + Protocol::MsgConnectionAck ack; + ack.Send( connection ); + } } // Process( MsgStatus ) From b2865314743bce0dc776fa5e99e87388243f8e59 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 26 Aug 2023 14:16:46 +0930 Subject: [PATCH 087/129] Fix issues compiling with VS2022 --- Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp | 2 +- External/Zstd/Zstd.bff | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp index d587fd95b..c05d005e8 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Server.cpp @@ -298,7 +298,7 @@ void Server::Process( const ConnectionInfo * connection, const Protocol::MsgConn if ( msg->GetProtocolVersionMinor() >= 3 ) { // Send Ack to client - Protocol::MsgConnectionAck ack; + const Protocol::MsgConnectionAck ack; ack.Send( connection ); } } diff --git a/External/Zstd/Zstd.bff b/External/Zstd/Zstd.bff index 6cc807851..dcca4da8d 100644 --- a/External/Zstd/Zstd.bff +++ b/External/Zstd/Zstd.bff @@ -11,12 +11,15 @@ .ZstdCompilerOptions = ' /wd4365' // conversion from '%s' to '%s', signed/unsigned mismatch + ' /wd4464' // relative include path contains '..' + ' /wd4574' // '__has_attribute' is defined to be '0': did you mean to use '#if __has_attribute'? + + ' /wd5262' // implicit fall-through occurs here; are you missing a break statement? + ' /wd6011' // Dereferencing NULL pointer 'ptr'. + ' /wd6239' // ( && ) always evaluates to the result of . Did you intend to use the bitwise-and operator? + ' /wd6293' // Ill-defined for-loop: counts down from minimum. + ' /wd6326' // Potential comparison of a constant with another constant. + + ' /wd26408' // Avoid malloc() and free(), prefer the nothrow version of new with delete (r.10). + ' /wd26448' // Consider using gsl::finally if final action is intended (gsl.util). + ' /wd26462' // The value pointed to by '%s' is assigned only once, mark it as a pointer to const + + ' /wd26496' // The variable '%s' does not change after construction, mark it as const (con.4). + ' /wd26818' // Switch statement does not cover all cases. Consider adding a 'default' label (es.79). + ' /wd26819' // Unannotated fallthrough between switch labels (es.78). + ' /wd28251' // Inconsistent annotation for '_setjmp': this instance has no annotations. See (0). From 468996eb6d2710b9e92007226d1fc77fa4ef9f27 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 26 Aug 2023 14:17:05 +0930 Subject: [PATCH 088/129] Improve Clang 11, 14 & 15 for undetected case - add missing placeholder variables --- External/SDK/Clang/Windows/Clang11.bff | 3 ++- External/SDK/Clang/Windows/Clang14.bff | 3 ++- External/SDK/Clang/Windows/Clang15.bff | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/External/SDK/Clang/Windows/Clang11.bff b/External/SDK/Clang/Windows/Clang11.bff index a0a2b51d4..3ed2990b2 100644 --- a/External/SDK/Clang/Windows/Clang11.bff +++ b/External/SDK/Clang/Windows/Clang11.bff @@ -29,7 +29,8 @@ Print( '----------------------------------------------------------------------' ) Print( '- Unable to auto-detect Clang - please specify installation manually -' ) Print( '----------------------------------------------------------------------' ) - .Clang11_BasePath = .Set_Path_Here // <-- Set path here + .Clang11_BasePath = .Set_Path_Here // <-- Set path here + .Clang11_Version = .Set_Version_Here // <-- Set version here #endif #endif #endif diff --git a/External/SDK/Clang/Windows/Clang14.bff b/External/SDK/Clang/Windows/Clang14.bff index 8f8777b11..8aad0a9dd 100644 --- a/External/SDK/Clang/Windows/Clang14.bff +++ b/External/SDK/Clang/Windows/Clang14.bff @@ -29,7 +29,8 @@ Print( '----------------------------------------------------------------------' ) Print( '- Unable to auto-detect Clang - please specify installation manually -' ) Print( '----------------------------------------------------------------------' ) - .Clang14_BasePath = .Set_Path_Here // <-- Set path here + .Clang14_BasePath = .Set_Path_Here // <-- Set path here + .Clang14_Version = .Set_Version_Here // <-- Set version here #endif #endif #endif diff --git a/External/SDK/Clang/Windows/Clang15.bff b/External/SDK/Clang/Windows/Clang15.bff index 0a335d6dc..bc887e346 100644 --- a/External/SDK/Clang/Windows/Clang15.bff +++ b/External/SDK/Clang/Windows/Clang15.bff @@ -29,7 +29,8 @@ Print( '----------------------------------------------------------------------' ) Print( '- Unable to auto-detect Clang - please specify installation manually -' ) Print( '----------------------------------------------------------------------' ) - .Clang15_BasePath = .Set_Path_Here // <-- Set path here + .Clang15_BasePath = .Set_Path_Here // <-- Set path here + .Clang15_Version = .Set_Version_Here // <-- Set version here #endif #endif #endif From 0edfdbca0a67a31bf85cfaa35d2d89cace9dbe69 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 16 Sep 2023 15:34:49 +0930 Subject: [PATCH 089/129] [Fix] Fix slow job processing with extremely large amounts of jobs - fix bad gap sequence in sorting algorithm - additionally add move semantics support to sort --- Code/Core/Containers/Sort.h | 29 +++++----- Code/Core/CoreTest/Tests/TestArray.cpp | 74 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 17 deletions(-) diff --git a/Code/Core/Containers/Sort.h b/Code/Core/Containers/Sort.h index 208b40228..88af7266a 100644 --- a/Code/Core/Containers/Sort.h +++ b/Code/Core/Containers/Sort.h @@ -35,32 +35,27 @@ class AscendingCompareDeref template < class T, class COMPARE > void ShellSort( T * begin, T * end, const COMPARE & compare ) { + // Ciura, Marcin (2001). "Best Increments for the Average Case of Shellsort". + static const size_t gaps[] = { 510774, 227011, 100894, 44842, 19930, 8858, 3937, 1750, 701, 301, 132, 57, 23, 10, 4, 1 }; + const size_t numItems = (size_t)( end - begin ); - size_t increment = 3; - while ( increment > 0 ) + for ( const size_t increment : gaps ) { + if ( increment > numItems ) + { + continue; + } + for ( size_t i = 0; i < numItems; i++ ) { size_t j = i; - T temp( begin[ i ] ); + T temp( Move( begin[ i ] ) ); while ( ( j >= increment ) && ( compare( temp, begin[ j - increment ] ) ) ) { - begin[ j ] = begin[ j - increment ]; + begin[ j ] = Move( begin[ j - increment ] ); j = j - increment; } - begin[ j ] = temp; - } - if ( increment / 2 != 0 ) - { - increment = increment / 2; - } - else if ( increment == 1 ) - { - increment = 0; - } - else - { - increment = 1; + begin[ j ] = Move( temp ); } } } diff --git a/Code/Core/CoreTest/Tests/TestArray.cpp b/Code/Core/CoreTest/Tests/TestArray.cpp index ea59fb9a9..4d452fe26 100644 --- a/Code/Core/CoreTest/Tests/TestArray.cpp +++ b/Code/Core/CoreTest/Tests/TestArray.cpp @@ -6,7 +6,9 @@ #include "TestFramework/TestGroup.h" #include "Core/Containers/Array.h" +#include "Core/Math/Random.h" #include "Core/Strings/AString.h" +#include "Core/Tracing/Tracing.h" // TestArray //------------------------------------------------------------------------------ @@ -37,6 +39,7 @@ class TestArray : public TestGroup void Sort() const; void SortDeref() const; + void SortBig() const; void Find() const; void FindDeref() const; @@ -106,6 +109,7 @@ REGISTER_TESTS_BEGIN( TestArray ) REGISTER_TEST( Sort ) REGISTER_TEST( SortDeref ) + REGISTER_TEST( SortBig ) REGISTER_TEST( Find ) REGISTER_TEST( FindDeref ) @@ -729,6 +733,76 @@ void TestArray::SortDeref() const } } +// SortBig +//------------------------------------------------------------------------------ +void TestArray::SortBig() const +{ + Random r; + + const uint32_t numItems = 1024 * 1024; + + // Ints + { + // Generate a set of random integers + Array bigArray; + bigArray.SetSize( numItems ); + for ( uint32_t & element : bigArray ) + { + element = r.GetRand(); + } + + // Sort + Timer t; + bigArray.Sort(); + const float t1 = t.GetElapsed(); + CheckConsistency( bigArray ); + OUTPUT( "SortBig 1, %2.3fs, %u\n", static_cast( t1 ), numItems ); + + // Validate ordering + for ( uint32_t j = 1; j < bigArray.GetSize(); ++j ) + { + TEST_ASSERT( bigArray[ j - 1 ] <= bigArray[ j ] ); + } + } + + // AString + { + // Generate a set of random strings + Array bigArray; + bigArray.SetSize( numItems ); + for ( AString & element : bigArray ) + { + element.SetReserved( 4 ); + element.SetLength( r.GetRandIndex( 16 ) ); + for ( char & c : element ) + { + c = static_cast( 32 + r.GetRandIndex( 127 - 32 ) ); + } + } + + TEST_MEMORY_SNAPSHOT(s1); + + // Sort + Timer t; + bigArray.Sort(); + const float t1 = t.GetElapsed(); + CheckConsistency( bigArray ); + + // Validate that no allocations were performed + // (all strings should have been moved) + TEST_EXPECT_ALLOCATION_EVENTS( s1, 0 ) + + OUTPUT( "SortBig 2, %2.3fs, %u\n", static_cast( t1 ), numItems ); + + // Validate ordering + for ( uint32_t j = 1; j < bigArray.GetSize(); ++j ) + { + TEST_ASSERT( bigArray[ j - 1 ].Compare( bigArray[ j ] ) <= 0 ); + } + } +} + + // Find //------------------------------------------------------------------------------ void TestArray::Find() const From 0e40d6cd7cac58bc9d8e9bb302e5ee36ee5b3fe6 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 16 Sep 2023 16:27:28 +0930 Subject: [PATCH 090/129] [Fix] Restore full stack size for threads that call cache plugins (regression in v1.11) --- Code/Core/Process/ThreadPool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Core/Process/ThreadPool.cpp b/Code/Core/Process/ThreadPool.cpp index 907aaa189..f433de3a2 100644 --- a/Code/Core/Process/ThreadPool.cpp +++ b/Code/Core/Process/ThreadPool.cpp @@ -124,7 +124,7 @@ ThreadPool::ThreadPoolThread::ThreadPoolThread( uint32_t threadId, // Start thread AStackString<> threadName; threadName.Format( "ThreadPool_%02u", threadId ); - m_Thread.Start( ThreadFuncWrapper, threadName.Get(), this ); + m_Thread.Start( ThreadFuncWrapper, threadName.Get(), this, MEGABYTE ); } //------------------------------------------------------------------------------ From 4072ef239b7868c703201df960114004228b776f Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 23 Sep 2023 11:16:07 +0930 Subject: [PATCH 091/129] [Fix] Fix memory leak when a corrupt cache item is retrieved - Add missing FreeMemory call - Re-order other failure case messages so they occur before memory is freed --- Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp index c81c1b039..e53e2ae16 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp @@ -1423,6 +1423,7 @@ bool ObjectNode::RetrieveFromCache( Job * job ) " - File: '%s'\n" " - Key : %s\n", m_Name.Get(), cacheFileName.Get() ); + cache->FreeMemory( cacheData, cacheDataSize ); return false; } const size_t uncompressedDataSize = buffer.GetDataSize(); @@ -1439,8 +1440,8 @@ bool ObjectNode::RetrieveFromCache( Job * job ) { if ( !buffer.ExtractFile( i, fileNames[ i ] ) ) { - cache->FreeMemory( cacheData, cacheDataSize ); FLOG_ERROR( "Failed to write local file during cache retrieval '%s'", fileNames[ i ].Get() ); + cache->FreeMemory( cacheData, cacheDataSize ); return false; } @@ -1450,8 +1451,8 @@ bool ObjectNode::RetrieveFromCache( Job * job ) // set the time on the local file if ( timeSetOK == false ) { - cache->FreeMemory( cacheData, cacheDataSize ); FLOG_ERROR( "Failed to set timestamp after cache hit. Error: %s Target: '%s'", LAST_ERROR_STR, fileNames[ i ].Get() ); + cache->FreeMemory( cacheData, cacheDataSize ); return false; } } From 63434bd9d95bfa28eef42fd426ae5b1210b02ae4 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 8 Oct 2023 13:44:27 +1030 Subject: [PATCH 092/129] [Improvement] VCXProject supports additional Android Game Development Extension options - Expose various options for controlling debugging with AGDE [Note] DB version changed --- .../docs/functions/vcxproject.html | 42 ++++++++++++++++++- .../Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 2 +- .../FBuildCore/Graph/VCXProjectNode.cpp | 10 +++++ .../FBuild/FBuildCore/Graph/VCXProjectNode.h | 10 +++++ .../FBuildCore/Helpers/VSProjectGenerator.cpp | 15 +++++-- .../FBuild/Integration/notepad++markup.xml | 2 +- Code/Tools/FBuild/Integration/usertype.dat | 10 +++++ 7 files changed, 84 insertions(+), 7 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html b/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html index ef830db49..f8f0430d2 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/vcxproject.html @@ -136,6 +136,16 @@

VCXProject

.PackagePath, // (optional) Path to package to debug .AdditionalSymbolSearchPaths, // (optional) Additional symbol search paths for debugging .AndroidApkLocation // (optional) Location of APK for Android Game Development Extension +.AndroidDebugComponent // (optional) Debug component for Android Game Development Extension +.AndroidDebugTarget // (optional) Debug target for Android Game Development Extension +.AndroidJdb // (optional) JDB for Android Game Development Extension +.AndroidLldbPostAttachCommands // (optional) LLDB post attach cmds for Android Game Development Extension +.AndroidLldbStartupCommands // (optional) LLDB startup cmds for Android Game Development Extension +.AndroidPostApkInstallCommands // (optional) Post APK install cmds for Android Game Development Extension +.AndroidPreApkInstallCommands // (optional) Pre APK install cmds for Android Game Development Extension +.AndroidSymbolDirectories // (optional) Symbol dirs for Android Game Development Extension +.AndroidWaitForDebugger // (optional) Wait for Debugger for Android Game Development Extension +.LaunchFlags // (optional) Launch flags for Android Game Development Extension // Misc .PlatformToolset // (optional) Specify PlatformToolset. @@ -576,7 +586,37 @@

VCXProject

Path to additional symbols to be used when debugging.

AndroidApkLocation - String - (Optional)

-

Location of APK for Android Game Development Extension

+

Location of APK for Android Game Development Extension.

+
+

AndroidDebugComponent - String - (Optional)

+

Debug component for Android Game Development Extension.

+
+

AndroidDebugTarget - String - (Optional)

+

Debug target for Android Game Development Extension.

+
+

AndroidJdb - String - (Optional)

+

JDB for Android Game Development Extension.

+
+

AndroidLldbPostAttachCommands - String - (Optional)

+

LLDB post attach commands for Android Game Development Extension.

+
+

AndroidLldbStartupCommands - String - (Optional)

+

LLDB startup commands for Android Game Development Extension.

+
+

AndroidPostApkInstallCommands - String - (Optional)

+

Post APK install commands for Android Game Development Extension.

+
+

AndroidPreApkInstallCommands - String - (Optional)

+

Pre APK install commands for Android Game Development Extension.

+
+

AndroidSymbolDirectories - String - (Optional)

+

Symbol dirs for Android Game Development Extension.

+
+

AndroidWaitForDebugger - String - (Optional)

+

Wait for Debugger for Android Game Development Extension.

+
+

LaunchFlags - String - (Optional)

+

Launch flags for Android Game Development Extension.

PlatformToolset - String - (Optional)

diff --git a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h index cee3065ba..a62175af7 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/NodeGraph.h @@ -64,7 +64,7 @@ class NodeGraphHeader } inline ~NodeGraphHeader() = default; - enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 171 }; + enum : uint8_t { NODE_GRAPH_CURRENT_VERSION = 172 }; bool IsValid() const; bool IsCompatibleVersion() const { return m_Version == NODE_GRAPH_CURRENT_VERSION; } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp index 7dd2eb9d6..eb14ae0b4 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.cpp @@ -70,6 +70,16 @@ REFLECT_STRUCT_BEGIN_BASE( VSProjectConfigBase ) REFLECT( m_PackagePath, "PackagePath", MetaInheritFromOwner() + MetaOptional() ) REFLECT( m_AdditionalSymbolSearchPaths, "AdditionalSymbolSearchPaths", MetaInheritFromOwner() + MetaOptional() ) REFLECT( m_AndroidApkLocation, "AndroidApkLocation", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidDebugComponent, "AndroidDebugComponent", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidDebugTarget, "AndroidDebugTarget", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidJdb, "AndroidJdb", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidLldbPostAttachCommands, "AndroidLldbPostAttachCommands", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidLldbStartupCommands, "AndroidLldbStartupCommands", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidPostApkInstallCommands, "AndroidPostApkInstallCommands", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidPreApkInstallCommands, "AndroidPreApkInstallCommands", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidSymbolDirectories, "AndroidSymbolDirectories", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_AndroidWaitForDebugger, "AndroidWaitForDebugger", MetaInheritFromOwner() + MetaOptional() ) + REFLECT( m_LaunchFlags, "LaunchFlags", MetaInheritFromOwner() + MetaOptional() ) REFLECT_END( VSProjectConfigBase ) REFLECT_STRUCT_BEGIN( VSProjectConfig, VSProjectConfigBase, MetaNone() ) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h index f63301957..48fdd6549 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/VCXProjectNode.h @@ -60,6 +60,16 @@ class VSProjectConfigBase : public Struct AString m_PackagePath; AString m_AdditionalSymbolSearchPaths; AString m_AndroidApkLocation; + AString m_AndroidDebugComponent; + AString m_AndroidDebugTarget; + AString m_AndroidJdb; + AString m_AndroidLldbPostAttachCommands; + AString m_AndroidLldbStartupCommands; + AString m_AndroidPostApkInstallCommands; + AString m_AndroidPreApkInstallCommands; + AString m_AndroidSymbolDirectories; + AString m_AndroidWaitForDebugger; + AString m_LaunchFlags; }; // VSProjectConfig diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp index 7268077e0..1037aa308 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/VSProjectGenerator.cpp @@ -297,10 +297,17 @@ const AString & VSProjectGenerator::GenerateVCXProj( const AString & projectFile WritePGItem( "NMakeReBuildCommandLine", config.m_ProjectRebuildCommand ); WritePGItem( "NMakeCleanCommandLine", config.m_ProjectCleanCommand ); } - if ( !config.m_AndroidApkLocation.IsEmpty() ) - { - WritePGItem( "AndroidApkLocation", config.m_AndroidApkLocation ); - } + WritePGItem( "AndroidApkLocation", config.m_AndroidApkLocation ); + WritePGItem( "AndroidDebugComponent", config.m_AndroidDebugComponent ); + WritePGItem( "AndroidDebugTarget", config.m_AndroidDebugTarget ); + WritePGItem( "AndroidJdb", config.m_AndroidJdb ); + WritePGItem( "AndroidLldbPostAttachCommands", config.m_AndroidLldbPostAttachCommands ); + WritePGItem( "AndroidLldbStartupCommands", config.m_AndroidLldbStartupCommands ); + WritePGItem( "AndroidPostApkInstallCommands", config.m_AndroidPostApkInstallCommands ); + WritePGItem( "AndroidPreApkInstallCommands", config.m_AndroidPreApkInstallCommands ); + WritePGItem( "AndroidSymbolDirectories", config.m_AndroidSymbolDirectories ); + WritePGItem( "AndroidWaitForDebugger", config.m_AndroidWaitForDebugger ); + WritePGItem( "LaunchFlags", config.m_LaunchFlags ); WritePGItem( "NMakeOutput", config.m_Output ); const ObjectListNode * oln = nullptr; diff --git a/Code/Tools/FBuild/Integration/notepad++markup.xml b/Code/Tools/FBuild/Integration/notepad++markup.xml index 696675c72..84b9b0876 100644 --- a/Code/Tools/FBuild/Integration/notepad++markup.xml +++ b/Code/Tools/FBuild/Integration/notepad++markup.xml @@ -25,7 +25,7 @@ Alias CSAssembly Compiler Copy CopyDir DLL Error Exec Executable ForEach If Library ListDependencies ObjectList Print RemoveDir Settings Test TextFile Unity Using VCXProject VSProjectExternal VSSolution XCodeProject - AdditionalOptions AdditionalSymbolSearchPaths AllowCaching AllowDistribution AllowResponseFile AndroidApkLocation ApplicationEnvironment ApplicationType ApplicationTypeRevision AssemblySearchPath AumidOverride BaseProjectConfig BaseSolutionConfig BuildLogFile CachePath CachePathMountPoint CachePluginDLL CachePluginDLLConfig ClangFixupUnity_Disable ClangGCCUpdateXLanguageArg ClangRewriteIncludes Compiler CompilerFamily CompilerForceUsing CompilerInputAllowNoFiles CompilerInputExcludePath CompilerInputExcludePattern CompilerInputExcludedFiles CompilerInputFile CompilerInputFiles CompilerInputFilesRoot CompilerInputObjectLists CompilerInputPath CompilerInputPathRecurse CompilerInputPattern CompilerInputUnity CompilerOptions CompilerOptionsDeoptimized CompilerOutput CompilerOutputExtension CompilerOutputKeepBaseExtension CompilerOutputPath CompilerOutputPrefix CompilerReferences Condition Config CustomEnvironmentVariables DebuggerFlavor DefaultLanguage DeoptimizeWritableFiles DeoptimizeWritableFilesWithToken Dependencies DeploymentFiles DeploymentType Dest DistributableJobMemoryLimitMiB Environment ExecAlways ExecAlwaysShowOutput ExecArguments ExecExecutable ExecInput ExecInputExcludePath ExecInputExcludePattern ExecInputExcludedFiles ExecInputPath ExecInputPathRecurse ExecInputPattern ExecOutput ExecReturnCode ExecUseStdOutAsOutput ExecWorkingDir Executable ExecutableRootPath ExternalProjectPath ExtraFiles FileType ForceResponseFile ForcedIncludes ForcedUsingAssemblies Hidden IncludeSearchPath IntermediateDirectory Items Keyword LayoutDir LayoutExtensionFilter Librarian LibrarianAdditionalInputs LibrarianAllowResponseFile LibrarianForceResponseFile LibrarianOptions LibrarianOutput LibrarianType Libraries Libraries2 Linker LinkerAllowResponseFile LinkerAssemblyResources LinkerForceResponseFile LinkerLinkObjects LinkerOptions LinkerOutput LinkerStampExe LinkerStampExeArgs LinkerType LinuxProjectType LocalDebuggerCommand LocalDebuggerCommandArguments LocalDebuggerEnvironment LocalDebuggerWorkingDirectory Output OutputDirectory PCHInputFile PCHObjectFileName PCHOptions PCHOutputFile PackagePath Path Pattern Patterns Platform PlatformToolset PreBuildDependencies Preprocessor PreprocessorDefinitions PreprocessorOptions Project ProjectAllowedFileExtensions ProjectBasePath ProjectBuildCommand ProjectCleanCommand ProjectConfigs ProjectFileTypes ProjectFiles ProjectFilesToExclude ProjectGuid ProjectInputPaths ProjectInputPathsExclude ProjectInputPathsRecurse ProjectOutput ProjectPatternToExclude ProjectProjectImports ProjectProjectReferences ProjectRebuildCommand ProjectReferences ProjectSccEntrySAK ProjectTypeGuid Projects RemoteDebuggerCommand RemoteDebuggerCommandArguments RemoteDebuggerWorkingDirectory RemoveExcludeFiles RemoveExcludePaths RemovePaths RemovePathsRecurse RemovePatterns RootNamespace SimpleDistributionMode SolutionBuildProject SolutionConfig SolutionConfigs SolutionDependencies SolutionDeployProjects SolutionFolders SolutionMinimumVisualStudioVersion SolutionOutput SolutionPlatform SolutionProjects SolutionVisualStudioVersion Source SourceExcludePaths SourceMapping_Experimental SourcePaths SourcePathsPattern SourcePathsRecurse Target TargetLinuxPlatform Targets TestAlwaysShowOutput TestArguments TestExecutable TestInput TestInputExcludePath TestInputExcludePattern TestInputExcludedFiles TestInputPath TestInputPathRecurse TestInputPattern TestOutput TestTimeOut TestWorkingDir TextFileAlways TextFileInputStrings TextFileOutput UnityInputExcludePath UnityInputExcludePattern UnityInputExcludedFiles UnityInputFiles UnityInputIsolateListFile UnityInputIsolateWritableFiles UnityInputIsolateWritableFilesLimit UnityInputIsolatedFiles UnityInputObjectLists UnityInputPath UnityInputPathRecurse UnityInputPattern UnityNumFiles UnityOutputPath UnityOutputPattern UnityPCH UseLightCache_Experimental UseRelativePaths_Experimental VS2012EnumBugFix WorkerConnectionLimit Workers XCodeBaseSDK XCodeBuildToolArgs XCodeBuildToolPath XCodeBuildWorkingDir XCodeCommandLineArguments XCodeCommandLineArgumentsDisabled XCodeDebugWorkingDir XCodeDocumentVersioning XCodeIphoneOSDeploymentTarget XCodeOrganizationName Xbox360DebuggerCommand + AdditionalOptions AdditionalSymbolSearchPaths AllowCaching AllowDistribution AllowResponseFile AndroidApkLocation AndroidDebugComponent AndroidDebugTarget AndroidJdb AndroidLldbPostAttachCommands AndroidLldbStartupCommands AndroidPostApkInstallCommands AndroidPreApkInstallCommands AndroidSymbolDirectories AndroidWaitForDebugger ApplicationEnvironment ApplicationType ApplicationTypeRevision AssemblySearchPath AumidOverride BaseProjectConfig BaseSolutionConfig BuildLogFile CachePath CachePathMountPoint CachePluginDLL CachePluginDLLConfig ClangFixupUnity_Disable ClangGCCUpdateXLanguageArg ClangRewriteIncludes Compiler CompilerFamily CompilerForceUsing CompilerInputAllowNoFiles CompilerInputExcludePath CompilerInputExcludePattern CompilerInputExcludedFiles CompilerInputFile CompilerInputFiles CompilerInputFilesRoot CompilerInputObjectLists CompilerInputPath CompilerInputPathRecurse CompilerInputPattern CompilerInputUnity CompilerOptions CompilerOptionsDeoptimized CompilerOutput CompilerOutputExtension CompilerOutputKeepBaseExtension CompilerOutputPath CompilerOutputPrefix CompilerReferences Condition Config CustomEnvironmentVariables DebuggerFlavor DefaultLanguage DeoptimizeWritableFiles DeoptimizeWritableFilesWithToken Dependencies DeploymentFiles DeploymentType Dest DistributableJobMemoryLimitMiB Environment ExecAlways ExecAlwaysShowOutput ExecArguments ExecExecutable ExecInput ExecInputExcludePath ExecInputExcludePattern ExecInputExcludedFiles ExecInputPath ExecInputPathRecurse ExecInputPattern ExecOutput ExecReturnCode ExecUseStdOutAsOutput ExecWorkingDir Executable ExecutableRootPath ExternalProjectPath ExtraFiles FileType ForceResponseFile ForcedIncludes ForcedUsingAssemblies Hidden IncludeSearchPath IntermediateDirectory Items Keyword LaunchFlags LayoutDir LayoutExtensionFilter Librarian LibrarianAdditionalInputs LibrarianAllowResponseFile LibrarianForceResponseFile LibrarianOptions LibrarianOutput LibrarianType Libraries Libraries2 Linker LinkerAllowResponseFile LinkerAssemblyResources LinkerForceResponseFile LinkerLinkObjects LinkerOptions LinkerOutput LinkerStampExe LinkerStampExeArgs LinkerType LinuxProjectType LocalDebuggerCommand LocalDebuggerCommandArguments LocalDebuggerEnvironment LocalDebuggerWorkingDirectory Output OutputDirectory PCHInputFile PCHObjectFileName PCHOptions PCHOutputFile PackagePath Path Pattern Patterns Platform PlatformToolset PreBuildDependencies Preprocessor PreprocessorDefinitions PreprocessorOptions Project ProjectAllowedFileExtensions ProjectBasePath ProjectBuildCommand ProjectCleanCommand ProjectConfigs ProjectFileTypes ProjectFiles ProjectFilesToExclude ProjectGuid ProjectInputPaths ProjectInputPathsExclude ProjectInputPathsRecurse ProjectOutput ProjectPatternToExclude ProjectProjectImports ProjectProjectReferences ProjectRebuildCommand ProjectReferences ProjectSccEntrySAK ProjectTypeGuid Projects RemoteDebuggerCommand RemoteDebuggerCommandArguments RemoteDebuggerWorkingDirectory RemoveExcludeFiles RemoveExcludePaths RemovePaths RemovePathsRecurse RemovePatterns RootNamespace SimpleDistributionMode SolutionBuildProject SolutionConfig SolutionConfigs SolutionDependencies SolutionDeployProjects SolutionFolders SolutionMinimumVisualStudioVersion SolutionOutput SolutionPlatform SolutionProjects SolutionVisualStudioVersion Source SourceExcludePaths SourceMapping_Experimental SourcePaths SourcePathsPattern SourcePathsRecurse Target TargetLinuxPlatform Targets TestAlwaysShowOutput TestArguments TestExecutable TestInput TestInputExcludePath TestInputExcludePattern TestInputExcludedFiles TestInputPath TestInputPathRecurse TestInputPattern TestOutput TestTimeOut TestWorkingDir TextFileAlways TextFileInputStrings TextFileOutput UnityInputExcludePath UnityInputExcludePattern UnityInputExcludedFiles UnityInputFiles UnityInputIsolateListFile UnityInputIsolateWritableFiles UnityInputIsolateWritableFilesLimit UnityInputIsolatedFiles UnityInputObjectLists UnityInputPath UnityInputPathRecurse UnityInputPattern UnityNumFiles UnityOutputPath UnityOutputPattern UnityPCH UseLightCache_Experimental UseRelativePaths_Experimental VS2012EnumBugFix WorkerConnectionLimit Workers XCodeBaseSDK XCodeBuildToolArgs XCodeBuildToolPath XCodeBuildWorkingDir XCodeCommandLineArguments XCodeCommandLineArgumentsDisabled XCodeDebugWorkingDir XCodeDocumentVersioning XCodeIphoneOSDeploymentTarget XCodeOrganizationName Xbox360DebuggerCommand ) %1 %2 %3 diff --git a/Code/Tools/FBuild/Integration/usertype.dat b/Code/Tools/FBuild/Integration/usertype.dat index aaa4dbb4d..b49adbc56 100644 --- a/Code/Tools/FBuild/Integration/usertype.dat +++ b/Code/Tools/FBuild/Integration/usertype.dat @@ -46,6 +46,15 @@ AllowCaching AllowDistribution AllowResponseFile AndroidApkLocation +AndroidDebugComponent +AndroidDebugTarget +AndroidJdb +AndroidLldbPostAttachCommands +AndroidLldbStartupCommands +AndroidPostApkInstallCommands +AndroidPreApkInstallCommands +AndroidSymbolDirectories +AndroidWaitForDebugger ApplicationEnvironment ApplicationType ApplicationTypeRevision @@ -125,6 +134,7 @@ IncludeSearchPath IntermediateDirectory Items Keyword +LaunchFlags LayoutDir LayoutExtensionFilter Librarian From c7b262e45eea834b0ddf161bbcaeae155627e6ad Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 15 Oct 2023 13:01:16 +1030 Subject: [PATCH 093/129] Add logic to differentiate between Performance and Efficiency cores - not used yet, but once tested can be used to improve thread pool logic --- Code/Core/CoreTest/Tests/TestEnv.cpp | 14 +++++ Code/Core/Env/Env.cpp | 76 ++++++++++++++++++++++++++++ Code/Core/Env/Env.h | 10 ++++ 3 files changed, 100 insertions(+) diff --git a/Code/Core/CoreTest/Tests/TestEnv.cpp b/Code/Core/CoreTest/Tests/TestEnv.cpp index a6b2c6cbd..79268c8b3 100644 --- a/Code/Core/CoreTest/Tests/TestEnv.cpp +++ b/Code/Core/CoreTest/Tests/TestEnv.cpp @@ -8,6 +8,7 @@ // Core #include #include +#include // TestEnv //------------------------------------------------------------------------------ @@ -17,6 +18,7 @@ class TestEnv : public TestGroup DECLARE_TESTS void GetCommandLine() const; + void GetProcessorInfo() const; void GetExePath() const; }; @@ -24,6 +26,7 @@ class TestEnv : public TestGroup //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestEnv ) REGISTER_TEST( GetCommandLine ) + REGISTER_TEST( GetProcessorInfo ) REGISTER_TEST( GetExePath ) REGISTER_TESTS_END @@ -49,4 +52,15 @@ void TestEnv::GetExePath() const #endif } +// GetProcessorInfo +//------------------------------------------------------------------------------ +void TestEnv::GetProcessorInfo() const +{ + const Env::ProcessorInfo& info = Env::GetProcessorInfo(); + OUTPUT( "Num Cores: %u (PCores: %u + ECores: %u)\n", + info.mNumCores, + info.mNumPCores, + info.mNumECores ); +} + //------------------------------------------------------------------------------ diff --git a/Code/Core/Env/Env.cpp b/Code/Core/Env/Env.cpp index 36a6955de..c68d21f12 100644 --- a/Code/Core/Env/Env.cpp +++ b/Code/Core/Env/Env.cpp @@ -7,6 +7,7 @@ // Core #include "Core/Containers/Array.h" +#include "Core/Profile/Profile.h" #include "Core/Process/Atomic.h" #include "Core/Strings/AStackString.h" @@ -36,6 +37,81 @@ }; #endif +// Env::ProcessorInfo +//------------------------------------------------------------------------------ +Env::ProcessorInfo::ProcessorInfo() +{ + PROFILE_FUNCTION; + + // Detect total core count + mNumCores = GetNumProcessors(); + + // Detect Performance and Efficiency core breakdown +#if defined( __WINDOWS__ ) + // On systems with NUMA nodes, assume all cores are Performance cores + // and use the existing detection logic which can returns CPU counts + // greater than 64 + ULONG numNUMANodes = 0; + VERIFY( GetNumaHighestNodeNumber( &numNUMANodes ) ); + if ( numNUMANodes > 0 ) + { + mNumPCores = mNumCores; + return; + } + + // Introspect each core per Intel's guidance: + // - https://www.intel.com/content/www/us/en/developer/articles/guide/12th-gen-intel-core-processor-gamedev-guide.html + // On AMD CPUs, all cores are detected as PCores currently as expected. + // If AMD starts shipping SKUs with ECores, this function may require additional updates/logic. + + // Take note of the original affinity mask so we can restore it + const uint64_t originalAffinity = SetThreadAffinityMask( GetCurrentThread(), 0xFFFFFFFFFFFFFFFFULL ); + ASSERT( originalAffinity ); + + // Iterate all the cores + for (uint32_t core = 0; core < mNumCores; ++core) + { + // Switch affinity to core so cpuid function returns info about that core + VERIFY( SetThreadAffinityMask( GetCurrentThread(), ( 1ULL << core ) ) != 0 ); + + // Check if the core is on a "hybrid part" + bool isECore = false; + int32_t cpuIdInfo[ 4 ]; + __cpuid( cpuIdInfo, 0x07 ); // Hybrid Part + if ( cpuIdInfo[3] & ( 1 << 15 ) ) // Bit 15 in EDX + { + // Query core type + __cpuid( cpuIdInfo, 0x07 ); // Core Type + + // Determine if this core is a PCore or ECore by checking + // the top 8 bits in EAX + if ( ( static_cast( cpuIdInfo[0]) >> 24 ) == 0x20 ) // Intel Atom + { + isECore = true; + } + } + + // Increment the appropriate core type + uint32_t& coreCount = ( isECore ? mNumECores : mNumPCores ); + coreCount++; + } + + // Restore original affinity + VERIFY( SetThreadAffinityMask( GetCurrentThread(), originalAffinity ) != 0 ); +#else + // TODO:LINUX TODO:OSX Implement for these platforms + mNumPCores = mNumCores; +#endif +} + +// GetProcessorInfo +//------------------------------------------------------------------------------ +/*static*/ const Env::ProcessorInfo& Env::GetProcessorInfo() +{ + static ProcessorInfo sInfo; // Info is gathered on first call + return sInfo; +} + // GetNumProcessors //------------------------------------------------------------------------------ /*static*/ uint32_t Env::GetNumProcessors() diff --git a/Code/Core/Env/Env.h b/Code/Core/Env/Env.h index ecff109a4..f1b0c1275 100644 --- a/Code/Core/Env/Env.h +++ b/Code/Core/Env/Env.h @@ -28,6 +28,16 @@ class Env static inline const char * GetPlatformName( Platform platform ); static inline const char * GetPlatformName() { return GetPlatformName( GetPlatform() ); } + class ProcessorInfo + { + public: + ProcessorInfo(); + + uint32_t mNumCores = 0; // Logical CPU cores + uint32_t mNumPCores = 0; // "Performance" cores + uint32_t mNumECores = 0; // "Efficiency" cores + }; + static const ProcessorInfo& GetProcessorInfo(); static uint32_t GetNumProcessors(); static bool GetEnvVariable( const char * envVarName, AString & envVarValue ); From 92d8f2eecb6d57517b3c3a125c4acf89d4e4ddd6 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 21 Oct 2023 07:59:02 +1030 Subject: [PATCH 094/129] Fix GetProcessorInfo detection of hybrid core types --- Code/Core/Env/Env.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/Core/Env/Env.cpp b/Code/Core/Env/Env.cpp index c68d21f12..bbef7909e 100644 --- a/Code/Core/Env/Env.cpp +++ b/Code/Core/Env/Env.cpp @@ -81,7 +81,7 @@ Env::ProcessorInfo::ProcessorInfo() if ( cpuIdInfo[3] & ( 1 << 15 ) ) // Bit 15 in EDX { // Query core type - __cpuid( cpuIdInfo, 0x07 ); // Core Type + __cpuid( cpuIdInfo, 0x1A ); // Core Type // Determine if this core is a PCore or ECore by checking // the top 8 bits in EAX From e9ce22c5bed5cd8caadd376ec900ed872b37d239 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 2 Dec 2023 12:55:05 +1030 Subject: [PATCH 095/129] [Fix] Fix spurious "Failed to spawn" errors when a build is being aborted - If a task was being spawned just as we were aborting a build, we would not correctly supress the errors caused by terminating the process --- Code/Core/Process/Process.cpp | 16 ++++++++++------ Code/Core/Process/Process.h | 3 +-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Code/Core/Process/Process.cpp b/Code/Core/Process/Process.cpp index d843970af..a49931dbf 100644 --- a/Code/Core/Process/Process.cpp +++ b/Code/Core/Process/Process.cpp @@ -51,7 +51,6 @@ Process::Process( const volatile bool * mainAbortFlag, , m_ChildPID( -1 ) , m_HasAlreadyWaitTerminated( false ) #endif - , m_HasAborted( false ) , m_MainAbortFlag( mainAbortFlag ) , m_AbortFlag( abortFlag ) { @@ -474,7 +473,7 @@ int32_t Process::WaitForExit() DWORD exitCode = 0; - if ( m_HasAborted == false ) + if ( HasAborted() == false ) { // Don't wait if using jobs and the process has been aborted. // It will be killed along with the fbuild process if the TerminateProcess has failed for any reason and @@ -582,13 +581,10 @@ bool Process::ReadAllData( AString & outMem, bool processExited = false; for ( ;; ) { - const bool mainAbort = ( m_MainAbortFlag && AtomicLoadRelaxed( m_MainAbortFlag ) ); - const bool abort = ( m_AbortFlag && AtomicLoadRelaxed( m_AbortFlag ) ); - if ( abort || mainAbort ) + if ( HasAborted() ) { PROFILE_SECTION( "Abort" ); KillProcessTree(); - m_HasAborted = true; break; } @@ -768,6 +764,14 @@ bool Process::ReadAllData( AString & outMem, #endif } +// HasAborted +//------------------------------------------------------------------------------ +bool Process::HasAborted() const +{ + return ( m_MainAbortFlag && AtomicLoadRelaxed( m_MainAbortFlag ) ) || + ( m_AbortFlag && AtomicLoadRelaxed( m_AbortFlag ) ); +} + // Terminate //------------------------------------------------------------------------------ void Process::Terminate() diff --git a/Code/Core/Process/Process.h b/Code/Core/Process/Process.h index 8e9735fef..f730bc8a1 100644 --- a/Code/Core/Process/Process.h +++ b/Code/Core/Process/Process.h @@ -39,7 +39,7 @@ class Process // Prevent handles being redirected void DisableHandleRedirection() { m_RedirectHandles = false; } #endif - [[nodiscard]] bool HasAborted() const { return m_HasAborted; } + [[nodiscard]] bool HasAborted() const; [[nodiscard]] static uint32_t GetCurrentId(); private: @@ -86,7 +86,6 @@ class Process int m_StdOutRead; int m_StdErrRead; #endif - bool m_HasAborted; const volatile bool * m_MainAbortFlag; // This member is set when we must cancel processes asap when the main process dies. const volatile bool * m_AbortFlag; }; From 97df7de254dcd8b6e7221093975aed352ae2d8f4 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sat, 9 Dec 2023 12:23:19 +1030 Subject: [PATCH 096/129] VS2022: Compile-time fixes for VS2022 17.8.3 - missing const - add asserts to catch Array overflows (C26831) even if warning was ultimately disabled due to being impractical - disable warning 5267 about "definition of implicit copy constructor for '%s' is deprecated". Further investigation will be needed to determine if this is worth fixing --- Code/Core/Containers/Array.h | 2 ++ Code/Core/CoreTest/Tests/TestArray.cpp | 4 ++-- External/SDK/VisualStudio/VS2022.bff | 30 +++++++++++++++----------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Code/Core/Containers/Array.h b/Code/Core/Containers/Array.h index 65cb69cd0..a1e490122 100644 --- a/Code/Core/Containers/Array.h +++ b/Code/Core/Containers/Array.h @@ -676,6 +676,7 @@ void Array< T >::Grow() const size_t currentCapacity = GetCapacity(); const size_t size = GetSize(); const size_t newCapacity = ( currentCapacity + ( currentCapacity >> 1 ) + 1 ); + ASSERT( newCapacity <= CAPACITY_MASK ); T * newMem = Allocate( newCapacity ); T * src = m_Begin; @@ -700,6 +701,7 @@ template < class T > T * Array< T >::Allocate( size_t numElements ) const { ASSERT( m_Resizeable ); + ASSERT( numElements <= CAPACITY_MASK ); constexpr size_t align = __alignof( T ) > sizeof( void * ) ? __alignof( T ) : sizeof( void * ); return static_cast< T * >( ALLOC( sizeof( T ) * numElements, align ) ); } diff --git a/Code/Core/CoreTest/Tests/TestArray.cpp b/Code/Core/CoreTest/Tests/TestArray.cpp index 4d452fe26..22e6113de 100644 --- a/Code/Core/CoreTest/Tests/TestArray.cpp +++ b/Code/Core/CoreTest/Tests/TestArray.cpp @@ -752,7 +752,7 @@ void TestArray::SortBig() const } // Sort - Timer t; + const Timer t; bigArray.Sort(); const float t1 = t.GetElapsed(); CheckConsistency( bigArray ); @@ -783,7 +783,7 @@ void TestArray::SortBig() const TEST_MEMORY_SNAPSHOT(s1); // Sort - Timer t; + const Timer t; bigArray.Sort(); const float t1 = t.GetElapsed(); CheckConsistency( bigArray ); diff --git a/External/SDK/VisualStudio/VS2022.bff b/External/SDK/VisualStudio/VS2022.bff index 3a6541dcd..288dc2bb7 100644 --- a/External/SDK/VisualStudio/VS2022.bff +++ b/External/SDK/VisualStudio/VS2022.bff @@ -8,13 +8,13 @@ // 2) Specified via environment variables (VS Command Prompt) // 3) Part of a Visual Studio installation (Program Files) // -#if file_exists( "2022/Community/VC/Tools/MSVC/14.35.32215/bin/Hostx64/x64/cl.exe" ) +#if file_exists( "2022/Community/VC/Tools/MSVC/14.38.33130/bin/Hostx64/x64/cl.exe" ) // - // Use vendorized toolchain + // Use vendorized toolchain - v17.8.3 // .VS2022_BasePath = '$_CURRENT_BFF_DIR_$/2022/Community/VC' - .VS2022_Version = '14.35.32215' - .VS2022_MSC_VER = '1935' + .VS2022_Version = '14.38.33130' + .VS2022_MSC_VER = '1938' #else // // Use environment variable if available (VS Command Prompt) @@ -24,25 +24,25 @@ #import VCToolsVersion .VS2022_BasePath = .VCINSTALLDIR .VS2022_Version = .VCToolsVersion - .VS2022_MSC_VER = '1935' // NOTE: This cannot be detected and may be incorrect + .VS2022_MSC_VER = '1938' // NOTE: This cannot be detected and may be incorrect #else // // Use Visual Studio installation // - #if file_exists( "C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/MSVC/14.35.32215/bin/Hostx64/x64/cl.exe" ) - // v17.1.4 + #if file_exists( "C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/MSVC/14.38.33130/bin/Hostx64/x64/cl.exe" ) + // v17.8.3 .VS2022_BasePath = 'C:/Program Files/Microsoft Visual Studio/2022/Community/VC' - .VS2022_Version = '14.35.32215' - .VS2022_MSC_VER = '1935' + .VS2022_Version = '14.38.33130' + .VS2022_MSC_VER = '1938' #else // // Use Visual Studio 2022 Enterprise installation (used by GitHub Actions) // - #if file_exists( "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.35.32215/bin/Hostx64/x64/cl.exe" ) - // v17.1.4 + #if file_exists( "C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC/Tools/MSVC/14.38.33130/bin/Hostx64/x64/cl.exe" ) + // v17.8.3 .VS2022_BasePath = 'C:/Program Files/Microsoft Visual Studio/2022/Enterprise/VC' - .VS2022_Version = '14.35.32215' - .VS2022_MSC_VER = '1935' + .VS2022_Version = '14.38.33130' + .VS2022_MSC_VER = '1938' #else // // Failed @@ -114,6 +114,9 @@ Compiler( 'Compiler-VS2022-x64' ) + ' /Wall' // Enable all warnings (we'll disable those that are not useful) + ' /WX' // Warnings as errors + // These warnings need further investigation + + ' /wd5267' // definition of implicit copy constructor for '%s' is deprecated because it has a user-provided destructor + // These warnings are not useful + ' /wd4061' // enumerator '%s' in switch of enum '%s' is not explicitly handled by a case label + ' /wd4255' // '%s': no function prototype given: converting '()' to '(void)'' @@ -230,6 +233,7 @@ Compiler( 'Compiler-VS2022-x64' ) + ' /wd26492' // Don't use const_cast to cast away const or volatile (type.3). + ' /wd26493' // Don't use C-style casts (type.4). + ' /wd26826' // Don't use C-style variable arguments (f.55). + + ' /wd26831' // Allocation size might be the result of a numerical overflow. ] //------------------------------------------------------------------------------ From 665771967a358bbadb9f2c376034140c23bd9a74 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 19 Dec 2023 10:34:25 +1030 Subject: [PATCH 097/129] BFFVariable: Remove unnecesary nullptr check for varDst -varDst is "this" and cannot be nullptr --- Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp index b4090e2fb..0e035335d 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFVariable.cpp @@ -292,8 +292,7 @@ BFFVariable * BFFVariable::ConcatVarsRecurse( const AString & dstName, const BFF { // Mismatched - is there a supported conversion? - const bool dstIsEmpty = ( varDst == nullptr ) || - ( ( dstType == BFFVariable::VAR_ARRAY_OF_STRINGS ) && varDst->GetArrayOfStrings().IsEmpty() ) || + const bool dstIsEmpty = ( ( dstType == BFFVariable::VAR_ARRAY_OF_STRINGS ) && varDst->GetArrayOfStrings().IsEmpty() ) || ( ( dstType == BFFVariable::VAR_ARRAY_OF_STRUCTS ) && varDst->GetArrayOfStructs().IsEmpty() ); const bool srcIsEmpty = ( ( srcType == BFFVariable::VAR_ARRAY_OF_STRINGS ) && varSrc->GetArrayOfStrings().IsEmpty() ) || ( ( srcType == BFFVariable::VAR_ARRAY_OF_STRUCTS ) && varSrc->GetArrayOfStructs().IsEmpty() ); From 952f5f51b0b0704d600d7a141e066fc9ad64db46 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:12:09 +1030 Subject: [PATCH 098/129] Clang17 Windows: - add support for compiling FASTBuild on Windows using Clang17 --- .../Data/TestPrecompiledHeaders/fbuild.bff | 4 +- External/SDK/Clang/Clang.bff | 4 + External/SDK/Clang/Windows/Clang17.bff | 194 ++++++++++++++++++ 3 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 External/SDK/Clang/Windows/Clang17.bff diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff index e71150069..939133543 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff @@ -71,12 +71,12 @@ Executable( "PCHTest" ) .CompilerOptions + ' /Yu"PrecompiledHeader.h" /Fp"$PCHOutputFile$"' + ' "/ITools/FBuild/FBuildTest/Data/TestPrecompiledHeaders"' + ' -Wno-unused-macros' - #if USING_CLANG_10 || USING_CLANG_11 || USING_CLANG_14 || USING_CLANG_15 + #if USING_CLANG_10 || USING_CLANG_11 || USING_CLANG_14 || USING_CLANG_15 || USING_CLANG_17 + ' -Wno-anon-enum-enum-conversion' #endif .PCHOptions + ' "/ITools/FBuild/FBuildTest/Data/TestPrecompiledHeaders"' + ' -Wno-unused-macros' - #if USING_CLANG_10 || USING_CLANG_11 || USING_CLANG_14 || USING_CLANG_15 + #if USING_CLANG_10 || USING_CLANG_11 || USING_CLANG_14 || USING_CLANG_15 || USING_CLANG_17 + ' -Wno-anon-enum-enum-conversion' #endif diff --git a/External/SDK/Clang/Clang.bff b/External/SDK/Clang/Clang.bff index 86f15a586..707935c20 100644 --- a/External/SDK/Clang/Clang.bff +++ b/External/SDK/Clang/Clang.bff @@ -20,6 +20,7 @@ #define USING_CLANG_11 //#define USING_CLANG_14 //#define USING_CLANG_15 + //#define USING_CLANG_17 #endif // Activate @@ -74,6 +75,9 @@ #if USING_CLANG_15 #include "Windows/Clang15.bff" #endif + #if USING_CLANG_17 + #include "Windows/Clang17.bff" + #endif #endif //------------------------------------------------------------------------------ diff --git a/External/SDK/Clang/Windows/Clang17.bff b/External/SDK/Clang/Windows/Clang17.bff new file mode 100644 index 000000000..fa5325437 --- /dev/null +++ b/External/SDK/Clang/Windows/Clang17.bff @@ -0,0 +1,194 @@ +// Clang 17.x.x +//------------------------------------------------------------------------------ +// +// Detect Clang +// +// We search in the following locations, in order of preference: +// 1) Vendorized in External (side by side with this bff) +// 2) Part of a Visual Studio installation (Program Files) +// 3) Default install location +// +#if file_exists( "17.0.1/bin/clang-cl.exe" ) + // Vendorized + .Clang17_BasePath = '$_CURRENT_BFF_DIR_$/17.0.1' + .Clang17_Version = '17.0.1' +#else + #if file_exists( "C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/LLVM/x64/bin/clang-cl.exe" ) + // Installed with VS2022 + .Clang17_BasePath = 'C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Tools/LLVM/x64' + .Clang17_Version = '17.x.x' + #else + #if file_exists( "C:/Program Files/LLVM/bin/clang-cl.exe" ) + // Default Install + .Clang17_BasePath = 'C:/Program Files/LLVM' + .Clang17_Version = '17.x.x' + #else + // + // Failed + // + Print( '----------------------------------------------------------------------' ) + Print( '- Unable to auto-detect Clang - please specify installation manually -' ) + Print( '----------------------------------------------------------------------' ) + .Clang17_BasePath = .Set_Path_Here // <-- Set path here + .Clang17_Version = .Set_Version_Here // <-- Set version here + #endif + #endif +#endif + +// Print details of used version. TODO:C Move to Clang.bff +Print( 'Using Clang $Clang17_Version$ from $Clang17_BasePath$' ) + +// Compiler +//------------------------------------------------------------------------------ +Compiler( 'Compiler-Clang17' ) +{ + .Root = '$Clang17_BasePath$' + .ExtraFiles = { + '$Root$/bin/concrt140.dll' + '$Root$/bin/msvcp140.dll' + '$Root$/bin/vcruntime140.dll' + '$Root$/bin/vcruntime140_1.dll' + } + .Executable = '$Root$\bin\clang-cl.exe' + + // Allow tests to activate some experimental behavior + #if ENABLE_RELATIVE_PATHS + .UseRelativePaths_Experimental = true + #endif + #if ENABLE_SOURCE_MAPPING + .SourceMapping_Experimental = '/fastbuild-test-mapping' + #endif +} + +// Compiler +//------------------------------------------------------------------------------ +Compiler( 'Compiler-Clang17-NonCL' ) +{ + .Root = '$Clang17_BasePath$' + .ExtraFiles = { + '$Root$/bin/msvcp140.dll' + '$Root$/bin/vcruntime140.dll' + } + .Executable = '$Root$\bin\clang.exe' + + // Allow tests to activate some experimental behavior + #if ENABLE_RELATIVE_PATHS + .UseRelativePaths_Experimental = true + #endif + #if ENABLE_SOURCE_MAPPING + .SourceMapping_Experimental = '/fastbuild-test-mapping' + #endif +} + +// ToolChain +//------------------------------------------------------------------------------ +.ToolChain_Clang_Windows_Common = +[ + // Clang for Windows relies on the VS being present: + // - crt headers + // - crt libs/dlls + Using( .ToolChain_VS_Windows_X64 ) + + .Platform = 'x64Clang' + + // Librarian + .Librarian = '$Clang17_BasePath$\bin\llvm-ar.exe' + .LibrarianOptions = '/c echo %1 > %2' // 'rc "%2" "%1"' // NOTE: output must come first + + // Linker + .Linker = '$Clang17_BasePath$\bin\lld-link.exe' + .LinkerOptions = '/NODEFAULTLIB /WX /NOLOGO /INCREMENTAL:NO /OUT:"%2" "%1" /DEBUG' + + .VSLibPaths + + // Compiler Warnings + .CommonCompilerWarningOptions // Enable warnings + = ' -Wall -Wextra -Weverything' // All warnings + + ' -Werror -Wfatal-errors' // Warnings as fatal errors + + // Warnings that are not useful + + ' -Wno-#pragma-messages' // warning : %s [-W#pragma-messages] + + ' -Wno-c++98-compat-pedantic' // variadic macros are incompatible with C++98 + + ' -Wno-exit-time-destructors' // declaration requires an exit-time destructor + + ' -Wno-global-constructors' // declaration requires a global destructor + + ' -Wno-invalid-offsetof' // we get the offset of members in non-POD types + + ' -Wno-missing-prototypes' // no previous prototype for function '%s' + + ' -Wno-missing-variable-declarations' // no previous extern declaration for non-static variable '%s' + + ' -Wno-gnu-line-marker' // Clang complains about directives its own preprocessor generated + + ' -Wno-switch-enum' // Allow the use of "default" labels (we keep -Wswitch to ensure all cases are handled) + + // Warnings that fire but might be best to be fixed + + ' -Wno-cast-function-type-strict' // converts to incompatible function type + + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier + + ' -Wno-deprecated-copy-dtor' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor + + ' -Wno-missing-noreturn' // function '%s' could be declared with attribute 'noreturn' + + ' -Wno-old-style-cast' // use of old-style cast + + ' -Wno-unsafe-buffer-usage' // unsafe pointer arithmetic + + // File Extensions + .LibExtension = '.a' + .ExeExtension = '.exe' +] + +// ToolChain +//------------------------------------------------------------------------------ +.ToolChain_Clang_Windows = +[ + Using( .ToolChain_Clang_Windows_Common ) + + // Compiler Options + .Compiler = 'Compiler-Clang17' + .CommonCompilerOptions = ' -c' // Compile only + + ' /Z7' // Include debug info + + // Include paths + + ' -I"./"' + + .VSIncludePaths_ClangCl + + // x64 + + ' -m64' + + // No RTTI + + ' /GR-' + + // Warnings + + .CommonCompilerWarningOptions + + .CompilerOptions = ' /TP -o"%2" "%1" $CommonCompilerOptions$' + + ' /std:c++17' // Enable c++17 features + .CompilerOptionsC = ' /TC -o"%2" "%1" $CommonCompilerOptions$' + .PCHOptions = ' /TP $CommonCompilerOptions$ "%1" /Fo"%3" /Fp"%2" /Yc"PrecompiledHeader.h"' + + ' /std:c++17' // Enable c++17 features + + // Exception Control + .UseExceptions = ' /EHs' +] + +// ToolChain +//------------------------------------------------------------------------------ +.ToolChain_ClangNonCL_Windows = +[ + Using( .ToolChain_Clang_Windows_Common ) + + // Compiler Options + .Compiler = 'Compiler-Clang17-NonCL' + .CommonCompilerOptions = ' -c' // Compile only + + ' -g' // Include debug info + + // Include paths + + ' "-I./"' + + // x64 + + ' -m64' + + // No RTTI + + ' -fno-rtti' + + // Warnings + + .CommonCompilerWarningOptions + + .CompilerOptions = ' -x c++ -o"%2" "%1" $CommonCompilerOptions$' + + ' -std=c++17' // Enable c++17 features + .CompilerOptionsC = ' -x c -o"%2" "%1" $CommonCompilerOptions$' +] + +//------------------------------------------------------------------------------ From 705621d74b7ca84778a26cf582e3411d338bbdae Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:16:11 +1030 Subject: [PATCH 099/129] FASTBuild VS2022 & Clang17 - build using VS2022 and Clang17 on Windows by default - CI remain usnig older versions --- External/SDK/Clang/Clang.bff | 4 ++-- External/SDK/VisualStudio/VisualStudio.bff | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/External/SDK/Clang/Clang.bff b/External/SDK/Clang/Clang.bff index 707935c20..2c2e01a69 100644 --- a/External/SDK/Clang/Clang.bff +++ b/External/SDK/Clang/Clang.bff @@ -17,10 +17,10 @@ //#define USING_CLANG_8 //#define USING_CLANG_9 //#define USING_CLANG_10 - #define USING_CLANG_11 + //#define USING_CLANG_11 //#define USING_CLANG_14 //#define USING_CLANG_15 - //#define USING_CLANG_17 + #define USING_CLANG_17 #endif // Activate diff --git a/External/SDK/VisualStudio/VisualStudio.bff b/External/SDK/VisualStudio/VisualStudio.bff index c76f9551b..a631e8c47 100644 --- a/External/SDK/VisualStudio/VisualStudio.bff +++ b/External/SDK/VisualStudio/VisualStudio.bff @@ -5,8 +5,8 @@ // Select desired Visual Studio version //------------------------------------------------------------------------------ //#define USING_VS2017 -#define USING_VS2019 -//#define USING_VS2022 +//#define USING_VS2019 +#define USING_VS2022 // Activate //------------------------------------------------------------------------------ From 7addc2de74fa0d984a0acb35b6a1e98ab6c62c95 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:37:16 +1030 Subject: [PATCH 100/129] VS2022: Tidy up compiler warnings - remove disabling of warnings that is no longer necessary --- External/SDK/VisualStudio/VS2022.bff | 5 ----- 1 file changed, 5 deletions(-) diff --git a/External/SDK/VisualStudio/VS2022.bff b/External/SDK/VisualStudio/VS2022.bff index 288dc2bb7..0a45ecc05 100644 --- a/External/SDK/VisualStudio/VS2022.bff +++ b/External/SDK/VisualStudio/VS2022.bff @@ -119,7 +119,6 @@ Compiler( 'Compiler-VS2022-x64' ) // These warnings are not useful + ' /wd4061' // enumerator '%s' in switch of enum '%s' is not explicitly handled by a case label - + ' /wd4255' // '%s': no function prototype given: converting '()' to '(void)'' + ' /wd4514' // '%s': unreferenced inline function has been removed + ' /wd4577' // 'noexcept' used with no exception handling mode specified; termination on exception is not guaranteed. Specify /EHsc + ' /wd4625' // '%s' : copy constructor was implicitly defined as deleted @@ -189,12 +188,10 @@ Compiler( 'Compiler-VS2022-x64' ) + ' /analyze:external-' // Disable analysis of external headers // To investigate: These warnings might be useful and should be checked - + ' /wd6553' // The annotation for function 'RegOpenKeyExA' on _Param_(3) does not apply to a value type. + ' /wd26400' // Do not assign the result of an allocation or a function call with an owner return value to a raw pointer, use owner instead (i.11). + ' /wd26401' // Do not delete a raw pointer that is not an owner (i.11). + ' /wd26430' // Symbol '%s' is not tested for nullness on all paths (f.23).: Lines: ... + ' /wd26432' // If you define or delete any default operation in the type '%s', define or delete them all (c.21). - + ' /wd26434' // Function '%s' hides a non-virtual function '%s' (c.128). + ' /wd26436' // The type '%s' with a virtual function needs either public virtual or protected non-virtual destructor (c.35). + ' /wd26438' // Avoid 'goto' (es.76). + ' /wd26451' // Arithmetic overflow: Using operator '*' on a 4 byte value and then casting the result to a 8 byte value. Cast the value to the wider type before calling operator '*' to avoid overflow (io.2). @@ -204,9 +201,7 @@ Compiler( 'Compiler-VS2022-x64' ) + ' /wd26491' // Don't use static_cast downcasts (type.2). + ' /wd26494' // Variable '%s' is uninitialized. Always initialize an object (type.5). + ' /wd26495' // Variable '%s' is uninitialized. Always initialize a member variable (type.6). - + ' /wd26456' // Operator '%s' hides a non-virtual operator '%s' (c.128). + ' /wd26497' // The function '%s' could be marked constexpr if compile-time evaluation is desired (f.4). - + ' /wd26812' // The enum type '%s' is unscoped. Prefer 'enum class' over 'enum' (Enum.3). + ' /wd26814' // The const variable '%s' can be computed at compile-time. Consider using constexpr (con.5). // These warnings are not useful. They either: From a8dd2839d489e311f3f82e42fae79b67089da7cf Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 24 Dec 2023 13:24:41 +1030 Subject: [PATCH 101/129] AString: Add AppendList convenience function - concetenates an Array --- Code/Core/CoreTest/Tests/TestAString.cpp | 11 ++++++++++ Code/Core/Strings/AString.cpp | 27 ++++++++++++++++++++++++ Code/Core/Strings/AString.h | 3 +++ 3 files changed, 41 insertions(+) diff --git a/Code/Core/CoreTest/Tests/TestAString.cpp b/Code/Core/CoreTest/Tests/TestAString.cpp index d77d46c72..6c1918147 100644 --- a/Code/Core/CoreTest/Tests/TestAString.cpp +++ b/Code/Core/CoreTest/Tests/TestAString.cpp @@ -464,6 +464,17 @@ void TestAString::Concatenation() const a.Append( AStackString<>( "hello" ) ); TEST_ASSERT( a == "hello" ); } + + // Arrays + { + AStackString<> a; + StackArray strings; + strings.EmplaceBack( "One" ); + strings.EmplaceBack( "Two" ); + strings.EmplaceBack( "Three" ); + a.AppendList( strings ); + TEST_ASSERT( a == "One,Two,Three" ); + } } // EmbeddedNuls diff --git a/Code/Core/Strings/AString.cpp b/Code/Core/Strings/AString.cpp index 5afb0bc99..71560b6be 100644 --- a/Code/Core/Strings/AString.cpp +++ b/Code/Core/Strings/AString.cpp @@ -620,6 +620,33 @@ AString & AString::AppendFormat( MSVC_SAL_PRINTF const char * fmtString, ... ) return *this; } +// AppendList +//------------------------------------------------------------------------------ +AString & AString::AppendList( const Array & list, char separator ) +{ + // Ignore empty lists explicitly to simplify logic below + if ( list.IsEmpty() ) + { + return *this; + } + + // Determine how many separators will be needed + size_t numSeparatorsRemaining = ( list.GetSize() - 1 ); + + // Append items with separators + for ( const AString & item : list ) + { + Append( item ); + if ( numSeparatorsRemaining > 0 ) + { + Append( separator ); + --numSeparatorsRemaining; + } + } + + return *this; +} + // Replace ( char, char ) //------------------------------------------------------------------------------ uint32_t AString::Replace( char from, char to, uint32_t maxReplaces ) diff --git a/Code/Core/Strings/AString.h b/Code/Core/Strings/AString.h index 69d34a84d..60e71612d 100644 --- a/Code/Core/Strings/AString.h +++ b/Code/Core/Strings/AString.h @@ -67,10 +67,13 @@ class AString AString & operator += ( char c ); AString & operator += ( const char * string ); AString & operator += ( const AString & string ); + AString & Append( char c ) { return this->operator +=( c ); } + AString & Append( const char * string ) { return this->operator +=( string ); } AString & Append( const AString & string ) { return this->operator +=( string ); } AString & Append( const char * string, size_t len ); AString & Append( const char * start, const char * end ) { return Append( start, static_cast( end - start ) ); } AString & AppendFormat( MSVC_SAL_PRINTF const char * fmtString, ... ) FORMAT_STRING( 2, 3 ); + AString & AppendList( const Array & list, char separator = ',' ); // comparison [[nodiscard]] bool operator == ( const char * other ) const; From be251e760116454e50cdef69385861cc001c682b Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 24 Dec 2023 13:25:11 +1030 Subject: [PATCH 102/129] DirectoryList: Minor code cleanup - use new AppendList helper function - make check for valid node name use real node name generation function - move initialization of some bools to header - remove erroneous code comment --- .../FBuildCore/Graph/DirectoryListNode.cpp | 67 ++++++------------- .../FBuildCore/Graph/DirectoryListNode.h | 4 +- .../FBuild/FBuildCore/Graph/RemoveDirNode.cpp | 3 +- .../FBuild/FBuildCore/Graph/RemoveDirNode.h | 2 +- 4 files changed, 24 insertions(+), 52 deletions(-) diff --git a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp index 8b83a74c7..d851711f5 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.cpp @@ -104,8 +104,6 @@ class DirectoryListNodeGetFilesHelper : public GetFilesHelper //------------------------------------------------------------------------------ DirectoryListNode::DirectoryListNode() : Node( Node::DIRECTORY_LIST_NODE ) - , m_Recursive( true ) - , m_IncludeReadOnlyStatusInHash( false ) { m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_LastBuildTimeMs = 100; @@ -115,21 +113,23 @@ DirectoryListNode::DirectoryListNode() //------------------------------------------------------------------------------ /*virtual*/ bool DirectoryListNode::Initialize( NodeGraph & /*nodeGraph*/, const BFFToken * /*iter*/, const Function * /*function*/ ) { - // ensure name is correctly formatted - // path|[patterns]|recursive|readonlyflag|[excludePath] - ASSERT( m_Name.BeginsWith( m_Path ) ); - ASSERT( m_Name[ m_Path.GetLength() ] == '|' ); - ASSERT( m_Patterns.IsEmpty() || ( m_Name.Find( m_Patterns[ 0 ].Get() ) == m_Name.Get() + m_Path.GetLength() + 1 ) ); - ASSERT( ( m_Recursive && m_Name.Find( "|true|" ) ) || - ( !m_Recursive && m_Name.Find( "|false|" ) ) ); - ASSERT( ( m_IncludeReadOnlyStatusInHash && m_Name.Find( "|rw|" ) ) || - ( !m_IncludeReadOnlyStatusInHash && m_Name.Find( "||" ) ) ); - - // paths must have trailing slash - ASSERT( m_Path.EndsWith( NATIVE_SLASH ) ); - - // make sure exclusion path has trailing slash if provided #if defined( ASSERTS_ENABLED ) + // ensure name is correctly formatted + AStackString<> expectedName; + FormatName( m_Path, + &m_Patterns, + m_Recursive, + m_IncludeReadOnlyStatusInHash, + m_ExcludePaths, + m_FilesToExclude, + m_ExcludePatterns, + expectedName ); + ASSERT( m_Name == expectedName ); + + // paths must have trailing slash + ASSERT( m_Path.EndsWith( NATIVE_SLASH ) ); + + // make sure exclusion path has trailing slash if provided for ( const AString & excludePath : m_ExcludePaths ) { ASSERT( excludePath.EndsWith( NATIVE_SLASH ) ); @@ -158,52 +158,25 @@ DirectoryListNode::~DirectoryListNode() = default; AStackString<> patternString; if ( patterns ) { - const size_t numPatterns = patterns->GetSize(); - for ( size_t i=0; i 0 ) - { - patternString += '<'; - } - patternString += (*patterns)[ i ]; - } + patternString.AppendList( *patterns, '<' ); } result.Format( "%s|%s|%s|%s|", path.Get(), patternString.Get(), recursive ? "true" : "false", includeReadOnlyFlagInHash ? "rw" : ""); - const AString * const end = excludePaths.End(); - for ( const AString * it = excludePaths.Begin(); it!=end; ++it ) - { - const AString & excludePath = *it; - ASSERT( excludePath.EndsWith( NATIVE_SLASH ) ); - result += excludePath; - result += '<'; - } + result.AppendList( excludePaths, '<' ); if ( !excludeFiles.IsEmpty() ) { result += '|'; - const AString * const endFiles = excludeFiles.End(); - for ( const AString * itFiles = excludeFiles.Begin(); itFiles != endFiles; ++itFiles ) - { - const AString & excludedFile = *itFiles; - result += excludedFile; - result += '<'; - } + result.AppendList( excludeFiles, '<' ); } if ( !excludePatterns.IsEmpty() ) { result += '|'; - const AString * const endPatterns = excludePatterns.End(); - for ( const AString * itPatterns = excludePatterns.Begin(); itPatterns != endPatterns; ++itPatterns ) - { - const AString & excludedPattern = *itPatterns; - result += excludedPattern; - result += '<'; - } + result.AppendList( excludePatterns, '<' ); } } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h index 30899b860..fb7689b4c 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h @@ -52,8 +52,8 @@ class DirectoryListNode : public Node Array< AString > m_ExcludePaths; Array< AString > m_FilesToExclude; Array< AString > m_ExcludePatterns; - bool m_Recursive; - bool m_IncludeReadOnlyStatusInHash; + bool m_Recursive = true; + bool m_IncludeReadOnlyStatusInHash = false; // Internal State Array< FileIO::FileInfo > m_Files; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp index 56905ab8d..f9c0feb12 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.cpp @@ -29,7 +29,6 @@ REFLECT_END( RemoveDirNode ) //------------------------------------------------------------------------------ RemoveDirNode::RemoveDirNode() : Node( Node::REMOVE_DIR_NODE ) - , m_RemovePathsRecurse( true ) { m_ControlFlags = Node::FLAG_ALWAYS_BUILD; m_RemovePatterns.EmplaceBack( "*" ); @@ -52,7 +51,7 @@ RemoveDirNode::RemoveDirNode() function, m_RemovePaths, m_RemoveExcludePaths, - m_RemoveExcludeFiles, // unused FilesToExclude + m_RemoveExcludeFiles, Array< AString >(), // unused ExcludePatterns m_RemovePathsRecurse, false, // Don't include read-only status in hash diff --git a/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.h b/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.h index d1aeabfd4..0785032fc 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.h +++ b/Code/Tools/FBuild/FBuildCore/Graph/RemoveDirNode.h @@ -27,7 +27,7 @@ class RemoveDirNode : public Node // Exposed Properties Array< AString > m_RemovePaths; - bool m_RemovePathsRecurse; + bool m_RemovePathsRecurse = true; Array< AString > m_RemovePatterns; Array< AString > m_RemoveExcludePaths; Array< AString > m_RemoveExcludeFiles; From bd8df03dd341ecbaa9b80dc4f42f3bed7235d644 Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Sun, 24 Dec 2023 14:28:30 +1030 Subject: [PATCH 103/129] Misc code cleanup: - replace manual iteration with range-based fors - replace manual pointer operations with Array::GetIndexOf --- Code/Core/Mem/MemPoolBlock.cpp | 4 +-- Code/Core/Network/TCPConnectionPool.cpp | 7 ++-- Code/Core/Profile/ProfileManager.cpp | 7 ++-- .../FBuild/FBuildCore/BFF/BFFFileExists.cpp | 2 +- .../FBuildCore/BFF/Functions/Function.cpp | 19 +++------- .../FBuildCore/BFF/Functions/FunctionCopy.cpp | 22 +++++------- .../BFF/Functions/FunctionUsing.cpp | 6 +--- Code/Tools/FBuild/FBuildCore/FBuild.cpp | 9 ++--- Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp | 4 +-- .../FBuild/FBuildCore/Graph/CopyDirNode.cpp | 8 ++--- .../FBuild/FBuildCore/Graph/ExecNode.cpp | 4 +-- .../FBuild/FBuildCore/Graph/LibraryNode.cpp | 8 ++--- .../FBuild/FBuildCore/Graph/LinkerNode.cpp | 27 +++++--------- .../FBuildCore/Graph/ObjectListNode.cpp | 17 ++++----- .../FBuild/FBuildCore/Graph/ObjectNode.cpp | 13 +++---- .../FBuild/FBuildCore/Graph/UnityNode.cpp | 26 +++++++------- .../FBuildCore/Graph/XCodeProjectNode.cpp | 6 ++-- .../FBuildCore/Helpers/BuildProfiler.cpp | 2 +- .../FBuildCore/Helpers/Report/JSONReport.cpp | 5 +-- .../FBuildCore/Helpers/SLNGenerator.cpp | 35 ++++++++----------- .../FBuildCore/Helpers/ToolManifest.cpp | 23 ++++++------ .../FBuild/FBuildCore/Protocol/Client.cpp | 19 ++++------ .../FBuildCore/WorkerPool/JobQueueRemote.cpp | 8 ++--- .../FBuildTest/Tests/TestBuildFBuild.cpp | 6 ++-- .../FBuildTest/Tests/TestDistributed.cpp | 6 ++-- .../FBuildWorker/FBuildWorkerOptions.cpp | 4 +-- 26 files changed, 104 insertions(+), 193 deletions(-) diff --git a/Code/Core/Mem/MemPoolBlock.cpp b/Code/Core/Mem/MemPoolBlock.cpp index 1cabde8ed..ead92a595 100644 --- a/Code/Core/Mem/MemPoolBlock.cpp +++ b/Code/Core/Mem/MemPoolBlock.cpp @@ -35,10 +35,8 @@ MemPoolBlock::~MemPoolBlock() #endif // free pages - const void * const * end = m_Pages.End(); - for ( void ** it = m_Pages.Begin(); it != end; ++it ) + for ( void * page : m_Pages ) { - void * page = *it; FREE( page ); } } diff --git a/Code/Core/Network/TCPConnectionPool.cpp b/Code/Core/Network/TCPConnectionPool.cpp index 2e9829c00..ce07744e1 100644 --- a/Code/Core/Network/TCPConnectionPool.cpp +++ b/Code/Core/Network/TCPConnectionPool.cpp @@ -653,12 +653,9 @@ bool TCPConnectionPool::Broadcast( const void * data, size_t size ) bool result = true; - ConnectionInfo ** it = m_Connections.Begin(); - const ConnectionInfo * const * end = m_Connections.End(); - while ( it < end ) + for ( const ConnectionInfo * connection : m_Connections ) { - result &= Send( *it, data, size ); - it++; + result &= Send( connection, data, size ); } return result; } diff --git a/Code/Core/Profile/ProfileManager.cpp b/Code/Core/Profile/ProfileManager.cpp index d2d20ce33..8dd4214f0 100644 --- a/Code/Core/Profile/ProfileManager.cpp +++ b/Code/Core/Profile/ProfileManager.cpp @@ -240,14 +240,12 @@ ProfileEvent * ProfileEventBuffer::AllocateEventStorage() } // write all the events we have - const ProfileEventInfo * const end = infos.End(); AStackString< 8192 > buffer; const double freqMul = ( (double)Timer::GetFrequencyInvFloatMS() * 1000.0 ); if ( g_ProfileEventLog.IsOpen() ) { - for ( const ProfileEventInfo * it = infos.Begin(); it != end; ++it ) + for ( const ProfileEventInfo & info : infos ) { - const ProfileEventInfo & info = *it; uint64_t threadId = (uint64_t)info.m_ThreadId; char threadIsAsString[ 32 ]; @@ -319,9 +317,8 @@ ProfileEvent * ProfileEventBuffer::AllocateEventStorage() } } - for ( const ProfileEventInfo * it = infos.Begin(); it != end; ++it ) + for ( ProfileEventInfo & info : infos ) { - const ProfileEventInfo & info = *it; FDELETE[] info.m_Events; } } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/BFFFileExists.cpp b/Code/Tools/FBuild/FBuildCore/BFF/BFFFileExists.cpp index 471406e01..ff35b097c 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/BFFFileExists.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/BFFFileExists.cpp @@ -26,7 +26,7 @@ bool BFFFileExists::CheckFile( const AString & fileName ) if ( found ) { // Yes - return existing result - const size_t index = (size_t)( found - m_FileNames.Begin() ); + const size_t index = m_FileNames.GetIndexOf( found ); return m_FileExists[ index ]; } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp index 3aac50c63..78c24c603 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/Function.cpp @@ -549,11 +549,8 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, filesToExcludeCleaned.Append( cleanPath ); } - const AString * const end = paths.End(); - for ( const AString * it = paths.Begin(); it != end; ++it ) + for ( const AString & path : paths ) { - const AString & path = *it; - // get node for the dir we depend on AStackString<> name; DirectoryListNode::FormatName( path, @@ -702,10 +699,8 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, const char * inputVarName, Dependencies & nodes ) { - const AString * const end = files.End(); - for ( const AString * it = files.Begin(); it != end; ++it ) + for ( const AString & file : files ) { - const AString & file = *it; if (!GetFileNode( nodeGraph, iter, function, file, inputVarName, nodes )) { return false; // GetFileNode will have emitted an error @@ -723,11 +718,8 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, const char * inputVarName, Dependencies & nodes ) { - const AString * const end = objectLists.End(); - for ( const AString * it = objectLists.Begin(); it != end; ++it ) + for ( const AString & objectList : objectLists ) { - const AString & objectList = *it; - // get node for the dir we depend on Node * node = nodeGraph.FindNode( objectList ); if ( node == nullptr ) @@ -858,11 +850,10 @@ bool Function::GetNodeList( NodeGraph & nodeGraph, if ( n->GetType() == Node::ALIAS_NODE ) { const AliasNode * an = n->CastTo< AliasNode >(); - const Dependencies & aNodes = an->GetAliasedNodes(); - for ( const Dependency * it = aNodes.Begin(); it != aNodes.End(); ++it ) + for ( const Dependency & dep : an->GetAliasedNodes() ) { // TODO:C by passing as string we'll be looking up again for no reason - const AString & subName = it->GetNode()->GetName(); + const AString & subName = dep.GetNode()->GetName(); if ( !GetNodeList( nodeGraph, iter, function, propertyName, subName, nodes, allowCopyDirNodes, allowUnityNodes, allowRemoveDirNodes, allowCompilerNodes ) ) { diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp index 0cc86e29c..fc78f674f 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionCopy.cpp @@ -59,18 +59,13 @@ FunctionCopy::FunctionCopy() } // check sources are not paths + for ( const AString & srcFile : sources ) { - const AString * const end = sources.End(); - for ( const AString * it = sources.Begin(); it != end; ++it ) + // source must be a file, not a path + if ( PathUtils::IsFolderPath( srcFile ) ) { - const AString & srcFile( *it ); - - // source must be a file, not a path - if ( PathUtils::IsFolderPath( srcFile ) ) - { - Error::Error_1105_PathNotAllowed( funcStartIter, this, ".Source", srcFile ); - return false; - } + Error::Error_1105_PathNotAllowed( funcStartIter, this, ".Source", srcFile ); + return false; } } @@ -89,11 +84,10 @@ FunctionCopy::FunctionCopy() // get source node Array< Node * > srcNodes; { - const AString * const end = sources.End(); - for ( const AString * it = sources.Begin(); it != end; ++it ) + for ( const AString & source : sources ) { - Node * srcNode = nodeGraph.FindNode( *it ); + Node * srcNode = nodeGraph.FindNode( source ); if ( srcNode ) { if ( GetSourceNodes( funcStartIter, srcNode, srcNodes ) == false ) @@ -104,7 +98,7 @@ FunctionCopy::FunctionCopy() else { // source file not defined by use - assume an external file - srcNodes.Append( nodeGraph.CreateNode( *it, funcStartIter ) ); + srcNodes.Append( nodeGraph.CreateNode( source, funcStartIter ) ); } } } diff --git a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp index e703eb173..fc946f696 100644 --- a/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp +++ b/Code/Tools/FBuild/FBuildCore/BFF/Functions/FunctionUsing.cpp @@ -95,12 +95,8 @@ FunctionUsing::FunctionUsing() } const BFFVariable * sameNameMember = nullptr; - const Array< const BFFVariable * > & members = v->GetStructMembers(); - for ( const BFFVariable ** it = members.Begin(); - it != members.End(); - ++it ) + for ( const BFFVariable * member : v->GetStructMembers() ) { - const BFFVariable * member = ( *it ); if ( ( sameNameMember == nullptr ) && ( parentScope == false ) && ( member->GetName() == v->GetName() ) ) { // We have a struct member with the same name as the struct variable itself. diff --git a/Code/Tools/FBuild/FBuildCore/FBuild.cpp b/Code/Tools/FBuild/FBuildCore/FBuild.cpp index 2412e9c17..d16c68150 100644 --- a/Code/Tools/FBuild/FBuildCore/FBuild.cpp +++ b/Code/Tools/FBuild/FBuildCore/FBuild.cpp @@ -569,14 +569,12 @@ bool FBuild::ImportEnvironmentVar( const char * name, bool optional, AString & v } // check if the environment var was already imported - const EnvironmentVarAndHash * it = m_ImportedEnvironmentVars.Begin(); - const EnvironmentVarAndHash * const end = m_ImportedEnvironmentVars.End(); - while ( it < end ) + for ( const EnvironmentVarAndHash & envVar : m_ImportedEnvironmentVars ) { - if ( it->GetName() == name ) + if ( envVar.GetName() == name ) { // check if imported environment changed since last import - if ( it->GetHash() != hash ) + if ( envVar.GetHash() != hash ) { FLOG_ERROR( "Overwriting imported environment variable '%s' with a different value = '%s'", name, value.Get() ); @@ -586,7 +584,6 @@ bool FBuild::ImportEnvironmentVar( const char * name, bool optional, AString & v // skip registration when already imported with same hash value return true; } - it++; } // import new variable name with its hash value diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp index 9e15eb50c..825acc10f 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp @@ -275,10 +275,8 @@ bool CSNode::BuildArgs( Args & fullArgs ) const AStackString<> quote( "\"" ); - const AString * const end = tokens.End(); - for ( const AString * it = tokens.Begin(); it!=end; ++it ) + for ( const AString & token : tokens ) { - const AString & token = *it; if ( token.EndsWith( "%1" ) ) { // handle /Option:%1 -> /Option:A /Option:B /Option:C diff --git a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp index e2fa8ca2e..c4b9fff44 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/CopyDirNode.cpp @@ -99,16 +99,12 @@ CopyDirNode::~CopyDirNode() = default; { // Grab the files const DirectoryListNode * dln = dep.GetNode()->CastTo< DirectoryListNode >(); - const Array< FileIO::FileInfo > & files = dln->GetFiles(); - const FileIO::FileInfo * const fEnd = files.End(); - for ( const FileIO::FileInfo * fIt = files.Begin(); - fIt != fEnd; - ++fIt ) + for ( const FileIO::FileInfo & file : dln->GetFiles() ) { // Create a CopyFileNode for each dynamically discovered file // source file (full path) - const AString & srcFile = fIt->m_Name; + const AString & srcFile = file.m_Name; // source file (relative to base path) const AStackString<> srcFileRel( srcFile.Get() + dln->GetPath().GetLength() ); diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp index 3388cd6e5..4c8c87f62 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ExecNode.cpp @@ -291,10 +291,8 @@ void ExecNode::GetFullArgs(AString & fullArgs) const AStackString<> quote("\""); - const AString * const end = tokens.End(); - for (const AString * it = tokens.Begin(); it != end; ++it) + for ( const AString & token : tokens ) { - const AString & token = *it; if (token.EndsWith("%1")) { // handle /Option:%1 -> /Option:A /Option:B /Option:C diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp index 0a5ccbe54..456e8735f 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/LibraryNode.cpp @@ -244,10 +244,8 @@ bool LibraryNode::BuildArgs( Args & fullArgs ) const // objects instead of the libs const bool objectsInsteadOfLibs = ( m_LibrarianFlags & LIB_FLAG_LIB ) ? false : true; - const AString * const end = tokens.End(); - for ( const AString * it = tokens.Begin(); it!=end; ++it ) + for ( const AString & token : tokens ) { - const AString & token = *it; if ( token.EndsWith( "%1" ) ) { // handle /Option:%1 -> /Option:A /Option:B /Option:C @@ -369,10 +367,8 @@ bool LibraryNode::BuildArgs( Args & fullArgs ) const Array< AString > tokens; args.Tokenize( tokens ); - const AString* const end = tokens.End(); - for ( const AString * it = tokens.Begin(); it != end; ++it ) + for ( const AString & token : tokens ) { - const AString& token = *it; if ( LinkerNode::IsLinkerArg_MSVC( token, "WX" ) ) { flags |= LIB_FLAG_WARNINGS_AS_ERRORS_MSVC; diff --git a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp index 656580a2f..8e6a576e4 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/LinkerNode.cpp @@ -453,11 +453,8 @@ bool LinkerNode::BuildArgs( Args & fullArgs ) const Array< AString > tokens( 1024, true ); m_LinkerOptions.Tokenize( tokens ); - const AString * const end = tokens.End(); - for ( const AString * it = tokens.Begin(); it!=end; ++it ) + for ( const AString & token : tokens ) { - const AString & token = *it; - // %1 -> InputFiles const char * found = token.Find( "%1" ); if ( found ) @@ -768,10 +765,8 @@ void LinkerNode::GetAssemblyResourceFiles( Args & fullArgs, const AString & pre, bool optLBRFlag = false; bool orderFlag = false; - const AString * const end = tokens.End(); - for ( const AString * it=tokens.Begin(); it!=end; ++it ) + for ( const AString & token : tokens ) { - const AString & token = *it; if ( IsLinkerArg_MSVC( token, "DLL" ) ) { flags |= LinkerNode::LINK_FLAG_DLL; @@ -855,10 +850,8 @@ void LinkerNode::GetAssemblyResourceFiles( Args & fullArgs, const AString & pre, Array< AString > tokens; args.Tokenize( tokens ); - const AString * const end = tokens.End(); - for ( const AString * it=tokens.Begin(); it!=end; ++it ) + for ( AString & token : tokens ) { - AStackString<256> token( *it ); token.ToLower(); if ( ( token == "-shared" ) || @@ -1221,13 +1214,12 @@ void LinkerNode::GetImportLibName( const AString & args, AString & importLibName else { // filter specifically listed default libs - const AString * const endI = defaultLibsToIgnore.End(); - for ( const AString * itI = defaultLibsToIgnore.Begin(); itI != endI; ++itI ) + for ( const AString & defaultLibToIgnore : defaultLibsToIgnore ) { const AString * const endD = defaultLibs.End(); for ( AString * itD = defaultLibs.Begin(); itD != endD; ++itD ) { - if ( itI->CompareI( *itD ) == 0 ) + if ( defaultLibToIgnore.EqualsI( *itD ) ) { defaultLibs.Erase( itD ); break; @@ -1243,23 +1235,22 @@ void LinkerNode::GetImportLibName( const AString & args, AString & importLibName libPaths.Append( envLibPaths ); // convert libs to nodes - const AString * const endL = libs.End(); - for ( const AString * itL = libs.Begin(); itL != endL; ++itL ) + for ( const AString & lib : libs ) { bool found = false; // is the file a full path? - if ( ( itL->GetLength() > 2 ) && ( (*itL)[ 1 ] == ':' ) ) + if ( ( lib.GetLength() > 2 ) && ( lib[ 1 ] == ':' ) ) { // check file exists in current location - if ( !GetOtherLibrary( nodeGraph, iter, function, otherLibraries, AString::GetEmpty(), *itL, found ) ) + if ( !GetOtherLibrary( nodeGraph, iter, function, otherLibraries, AString::GetEmpty(), lib, found ) ) { return false; // GetOtherLibrary will have emitted error } } else { - if ( !GetOtherLibrary( nodeGraph, iter, function, otherLibraries, libPaths, *itL ) ) + if ( !GetOtherLibrary( nodeGraph, iter, function, otherLibraries, libPaths, lib ) ) { return false; // GetOtherLibrary will have emitted error } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp index 9c089bb8f..c4fbcf7c2 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectListNode.cpp @@ -361,15 +361,13 @@ ObjectListNode::~ObjectListNode() = default; const DirectoryListNode * dln = dep.GetNode()->CastTo< DirectoryListNode >(); const Array< FileIO::FileInfo > & files = dln->GetFiles(); m_DynamicDependencies.SetCapacity( m_DynamicDependencies.GetSize() + files.GetSize() ); - for ( Array< FileIO::FileInfo >::Iter fIt = files.Begin(); - fIt != files.End(); - fIt++ ) + for ( const FileIO::FileInfo & file : files ) { // Create the file node (or find an existing one) - Node * n = nodeGraph.FindNode( fIt->m_Name ); + Node * n = nodeGraph.FindNode( file.m_Name ); if ( n == nullptr ) { - n = nodeGraph.CreateNode( fIt->m_Name ); + n = nodeGraph.CreateNode( file.m_Name ); } else if ( n->IsAFile() == false ) { @@ -399,15 +397,12 @@ ObjectListNode::~ObjectListNode() = default; const UnityNode * un = dep.GetNode()->CastTo< UnityNode >(); // unity files - const Array< AString > & unityFiles = un->GetUnityFileNames(); - for ( Array< AString >::Iter it = unityFiles.Begin(); - it != unityFiles.End(); - it++ ) + for ( const AString & unityFile : un->GetUnityFileNames() ) { - Node * n = nodeGraph.FindNode( *it ); + Node * n = nodeGraph.FindNode( unityFile ); if ( n == nullptr ) { - n = nodeGraph.CreateNode( *it ); + n = nodeGraph.CreateNode( unityFile ); } else if ( n->IsAFile() == false ) { diff --git a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp index e53e2ae16..3225e5f12 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/ObjectNode.cpp @@ -288,14 +288,12 @@ ObjectNode::~ObjectNode() // convert includes to nodes m_DynamicDependencies.Clear(); m_DynamicDependencies.SetCapacity( m_Includes.GetSize() ); - for ( Array< AString >::ConstIter it = m_Includes.Begin(); - it != m_Includes.End(); - it++ ) + for ( const AString & include : m_Includes ) { - Node * fn = nodeGraph.FindNode( *it ); + Node * fn = nodeGraph.FindNode( include ); if ( fn == nullptr ) { - fn = nodeGraph.CreateNode( *it ); + fn = nodeGraph.CreateNode( include ); } else if ( fn->IsAFile() == false ) { @@ -963,11 +961,8 @@ bool ObjectNode::ProcessIncludesWithPreProcessor( Job * job ) Array< AString > tokens; args.Tokenize( tokens ); - const AString * const end = tokens.End(); - for ( const AString * it = tokens.Begin(); it != end; ++it ) + for ( const AString & token : tokens ) { - const AString & token = *it; - if ( IsCompilerArg_MSVC( token, "Zi" ) || IsCompilerArg_MSVC( token, "ZI" ) ) { if ( !flags.IsClangCl() ) // with clang-cl, Zi is an alias for /Z7, it does not produce PDBs diff --git a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp index 6f9953ae1..40b8c6aaf 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/UnityNode.cpp @@ -429,16 +429,15 @@ UnityNode::~UnityNode() } // write allocation of includes for this unity file - const UnityFileAndOrigin * const end = filesInThisUnity.End(); size_t numFilesActuallyIsolatedInThisUnity( 0 ); - for ( const UnityFileAndOrigin * file = filesInThisUnity.Begin(); file != end; ++file ) + for ( const UnityFileAndOrigin & file : filesInThisUnity ) { // files which are modified can optionally be excluded from the unity bool isolateThisFile = false; if ( ( m_MaxIsolatedFiles == 0 ) || ( numIsolated <= m_MaxIsolatedFiles ) ) { // is the file writable? - if ( file->IsIsolated() ) + if ( file.IsIsolated() ) { isolateThisFile = true; } @@ -447,25 +446,25 @@ UnityNode::~UnityNode() if ( isolateThisFile ) { // disable compilation of this file (comment it out) - m_IsolatedFiles.EmplaceBack( file->GetName(), file->GetDirListOrigin() ); + m_IsolatedFiles.EmplaceBack( file.GetName(), file.GetDirListOrigin() ); numFilesActuallyIsolatedInThisUnity++; } else if ( noUnity ) { // No unity makes us build all files individually // We still generate the unity.cpp the same way to avoid changing it unnecessarily - m_IsolatedFiles.EmplaceBack( file->GetName(), file->GetDirListOrigin() ); + m_IsolatedFiles.EmplaceBack( file.GetName(), file.GetDirListOrigin() ); } // Get relative file path AStackString<> relativePath; if ( m_UseRelativePaths_Experimental ) { - PathUtils::GetRelativePath( includeBasePath, file->GetName(), relativePath ); + PathUtils::GetRelativePath( includeBasePath, file.GetName(), relativePath ); } // write pragma showing cpp file being compiled to assist resolving compilation errors - AStackString<> buffer( m_UseRelativePaths_Experimental ? relativePath : file->GetName() ); + AStackString<> buffer( m_UseRelativePaths_Experimental ? relativePath : file.GetName() ); buffer.Replace( BACK_SLASH, FORWARD_SLASH ); // avoid problems with slashes in generated code #if defined( __LINUX__ ) output += "//"; // TODO:LINUX - Find how to avoid GCC spamming "note:" about use of pragma @@ -491,7 +490,7 @@ UnityNode::~UnityNode() } else { - output += file->GetName(); + output += file.GetName(); } output += "\"\r\n\r\n"; } @@ -636,14 +635,13 @@ bool UnityNode::GetFiles( Array< UnityFileAndOrigin > & files ) } } - const Dependency * const sEnd = m_StaticDependencies.End(); - for ( const Dependency * sIt = m_StaticDependencies.Begin(); sIt != sEnd; ++sIt ) + for ( const Dependency & dep : m_StaticDependencies ) { - const Node* node = sIt->GetNode(); + const Node* node = dep.GetNode(); if ( node->GetType() == Node::DIRECTORY_LIST_NODE ) { - DirectoryListNode * dirNode = sIt->GetNode()->CastTo< DirectoryListNode >(); + DirectoryListNode * dirNode = node->CastTo< DirectoryListNode >(); const FileIO::FileInfo * const filesEnd = dirNode->GetFiles().End(); // filter files in the dir list @@ -667,7 +665,7 @@ bool UnityNode::GetFiles( Array< UnityFileAndOrigin > & files ) } else if ( node->GetType() == Node::OBJECT_LIST_NODE ) { - const ObjectListNode * objListNode = sIt->GetNode()->CastTo< ObjectListNode >(); + const ObjectListNode * objListNode = node->CastTo< ObjectListNode >(); // iterate all the files in the object list Array< AString > objListFiles; @@ -759,7 +757,7 @@ void UnityNode::FilterForceIsolated( Array< UnityFileAndOrigin > & files, Array< } } - files.SetSize( (uint64_t)( writeIt - files.Begin() ) ); + files.SetSize( files.GetIndexOf( writeIt ) ); } diff --git a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp index fc82ab480..b84ae2273 100644 --- a/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp +++ b/Code/Tools/FBuild/FBuildCore/Graph/XCodeProjectNode.cpp @@ -193,12 +193,10 @@ XCodeProjectNode::~XCodeProjectNode() = default; for ( const FileIO::FileInfo & file : dln->GetFiles() ) { //filter the file by pattern - const AString * pit = m_PatternToExclude.Begin(); - const AString * const pend = m_PatternToExclude.End(); bool keep = true; - for ( ; pit != pend; ++pit ) + for ( const AString & pattern : m_PatternToExclude ) { - if ( PathUtils::IsWildcardMatch( pit->Get(), file.m_Name.Get() ) ) + if ( PathUtils::IsWildcardMatch( pattern.Get(), file.m_Name.Get() ) ) { keep = false; break; diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp index 49da73971..0d17b66e0 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/BuildProfiler.cpp @@ -139,7 +139,7 @@ bool BuildProfiler::SaveJSON( const FBuildOptions & options, const char * fileN // Remote Processing for ( const WorkerInfo & workerInfo : m_WorkerInfo ) { - const uint32_t workedPid = (uint32_t)( &workerInfo - m_WorkerInfo.Begin() ); + const uint32_t workedPid = static_cast( m_WorkerInfo.GetIndexOf( &workerInfo ) ); buffer.AppendFormat( "{\"name\":\"process_name\",\"ph\":\"M\",\"pid\":%u,\"tid\":0,\"args\":{\"name\":\"Worker: %s\"}},", workedPid, workerInfo.m_WorkerName.Get() ); for ( uint32_t i = 1000; i <= workerInfo.m_MaxThreadId; ++i ) { diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp index dacd9417c..025e52e9c 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/Report/JSONReport.cpp @@ -471,11 +471,8 @@ void JSONReport::DoCPUTimeByItem( const FBuildStats & stats ) // Result const Array< const Node * > & nodes = stats.GetNodesByTime(); - for ( const Node ** it = nodes.Begin(); - it != nodes.End(); - ++ it ) + for ( const Node * node : nodes ) { - const Node * node = *it; const float time = ( (float)node->GetProcessingTime() * 0.001f ); // ms to s const char * type = node->GetTypeName(); const char * name = node->GetName().Get(); diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/SLNGenerator.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/SLNGenerator.cpp index e500d117a..d316129f7 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/SLNGenerator.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/SLNGenerator.cpp @@ -111,10 +111,9 @@ void SLNGenerator::WriteProjectListings( const AString& solutionBasePath, { // Project Listings - const VSProjectBaseNode * const * projectsEnd = projects.End(); - for( VSProjectBaseNode ** it = projects.Begin() ; it != projectsEnd ; ++it ) + for ( const VSProjectBaseNode * project : projects ) { - AStackString<> projectPath( (*it)->GetName() ); + const AString & projectPath = project->GetName(); // get project base name only const char * lastSlash = projectPath.FindLast( NATIVE_SLASH ); @@ -130,13 +129,13 @@ void SLNGenerator::WriteProjectListings( const AString& solutionBasePath, #endif // retrieve projectGuid - AStackString<> projectGuid( (*it)->GetProjectGuid() ); + AStackString<> projectGuid( project->GetProjectGuid() ); // Visual Studio expects the GUID to be uppercase projectGuid.ToUpper(); // retrieve projectTypeGuid - AStackString<> projectTypeGuid( (*it)->GetProjectTypeGuid() ); + AStackString<> projectTypeGuid( project->GetProjectTypeGuid() ); // Visual Studio expects the GUID to be uppercase projectTypeGuid.ToUpper(); @@ -146,11 +145,10 @@ void SLNGenerator::WriteProjectListings( const AString& solutionBasePath, // Manage dependencies Array< AString > dependencyGUIDs( 64, true ); - const AString & fullProjectPath = (*it)->GetName(); for ( const SolutionDependency & deps : solutionDependencies ) { // is the set of deps relevant to this project? - if ( !deps.m_Projects.Find( fullProjectPath ) ) + if ( !deps.m_Projects.Find( projectPath ) ) { continue; } @@ -184,7 +182,7 @@ void SLNGenerator::WriteProjectListings( const AString& solutionBasePath, for ( const SolutionFolder & solutionFolder : solutionFolders ) { // this has to be done here to have the same order of declaration (like visual) - if ( solutionFolder.m_Projects.Find( (*it)->GetName() ) ) + if ( solutionFolder.m_Projects.Find( projectPath ) ) { // generate a guid for the solution folder AStackString<> solutionFolderGuid; @@ -286,12 +284,11 @@ void SLNGenerator::WriteSolutionConfigurationPlatforms( const Array< SolutionCon Write( "\tGlobalSection(SolutionConfigurationPlatforms) = preSolution\r\n" ); // Solution Configurations - const SolutionConfig * const end = solutionConfigs.End(); - for( const SolutionConfig * it = solutionConfigs.Begin() ; it != end ; ++it ) + for ( const SolutionConfig & solutionConfig : solutionConfigs ) { Write( "\t\t%s|%s = %s|%s\r\n", - it->m_SolutionConfig.Get(), it->m_SolutionPlatform.Get(), - it->m_SolutionConfig.Get(), it->m_SolutionPlatform.Get() ); + solutionConfig.m_SolutionConfig.Get(), solutionConfig.m_SolutionPlatform.Get(), + solutionConfig.m_SolutionConfig.Get(), solutionConfig.m_SolutionPlatform.Get() ); } Write( "\tEndGlobalSection\r\n" ); @@ -377,22 +374,20 @@ void SLNGenerator::WriteNestedProjects( const Array< AString > & solutionProject Write( "\tGlobalSection(NestedProjects) = preSolution\r\n" ); // Write every project to solution folder relationships - const AString * const solutionProjectsToFolderEnd = solutionProjectsToFolder.End(); - for( const AString * it = solutionProjectsToFolder.Begin() ; it != solutionProjectsToFolderEnd ; ++it ) + for ( const AString & folderRelationship : solutionProjectsToFolder ) { - Write( "%s", it->Get() ); + Write( "%s", folderRelationship.Get() ); } // Write every intermediate path - const AString * const solutionFolderPathsEnd = solutionFolderPaths.End(); - for( const AString * it = solutionFolderPaths.Begin() ; it != solutionFolderPathsEnd ; ++it ) + for( const AString & solutionFolderPath : solutionFolderPaths ) { // parse solution folder parent path AStackString<> solutionFolderParentGuid; - const char * lastSlash = it->FindLast( NATIVE_SLASH ); + const char * lastSlash = solutionFolderPath.FindLast( NATIVE_SLASH ); if ( lastSlash ) { - AStackString<> solutionFolderParentPath( it->Get(), lastSlash ); + AStackString<> solutionFolderParentPath( solutionFolderPath.Get(), lastSlash ); VSProjectGenerator::FormatDeterministicProjectGUID( solutionFolderParentGuid, solutionFolderParentPath ); } @@ -400,7 +395,7 @@ void SLNGenerator::WriteNestedProjects( const Array< AString > & solutionProject { // generate a guid for the solution folder AStackString<> solutionFolderGuid; - VSProjectGenerator::FormatDeterministicProjectGUID( solutionFolderGuid, *it ); + VSProjectGenerator::FormatDeterministicProjectGUID( solutionFolderGuid, solutionFolderPath ); solutionFolderGuid.ToUpper(); solutionFolderParentGuid.ToUpper(); diff --git a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp index 202c0c67d..20f630418 100644 --- a/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp +++ b/Code/Tools/FBuild/FBuildCore/Helpers/ToolManifest.cpp @@ -488,15 +488,14 @@ bool ToolManifest::GetSynchronizationStatus( uint32_t & syncDone, uint32_t & syn MutexHolder mh( m_Mutex ); // is completely synchronized? - const ToolManifestFile * const end = m_Files.End(); - for ( const ToolManifestFile * it = m_Files.Begin(); it != end; ++it ) + for ( const ToolManifestFile & file : m_Files ) { - syncTotal += it->GetUncompressedContentSize(); - if ( it->GetSyncState() == ToolManifestFile::SYNCHRONIZED ) + syncTotal += file.GetUncompressedContentSize(); + if ( file.GetSyncState() == ToolManifestFile::SYNCHRONIZED ) { - syncDone += it->GetUncompressedContentSize(); + syncDone += file.GetUncompressedContentSize(); } - else if ( it->GetSyncState() == ToolManifestFile::SYNCHRONIZING ) + else if ( file.GetSyncState() == ToolManifestFile::SYNCHRONIZING ) { synching = true; } @@ -523,12 +522,11 @@ void ToolManifest::CancelSynchronizingFiles() bool atLeastOneFileCancelled = false; // is completely synchronized? - const ToolManifestFile * const end = m_Files.End(); - for ( ToolManifestFile * it = m_Files.Begin(); it != end; ++it ) + for ( ToolManifestFile & file : m_Files ) { - if ( it->GetSyncState() == ToolManifestFile::SYNCHRONIZING ) + if ( file.GetSyncState() == ToolManifestFile::SYNCHRONIZING ) { - it->SetSyncState( ToolManifestFile::NOT_SYNCHRONIZED ); + file.SetSyncState( ToolManifestFile::NOT_SYNCHRONIZED ); atLeastOneFileCancelled = true; } } @@ -658,10 +656,9 @@ bool ToolManifest::ReceiveFileData( uint32_t fileId, f.SetSyncState( ToolManifestFile::SYNCHRONIZED ); // is completely synchronized? - const ToolManifestFile * const end = m_Files.End(); - for ( const ToolManifestFile * it = m_Files.Begin(); it != end; ++it ) + for ( const ToolManifestFile & file : m_Files ) { - if ( it->GetSyncState() != ToolManifestFile::SYNCHRONIZED ) + if ( file.GetSyncState() != ToolManifestFile::SYNCHRONIZED ) { // still some files to be received return true; // file stored ok diff --git a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp index f17490f3c..d7801b29e 100644 --- a/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp +++ b/Code/Tools/FBuild/FBuildCore/Protocol/Client.cpp @@ -74,13 +74,10 @@ Client::~Client() DIST_INFO( "Disconnected: %s\n", ss->m_RemoteName.Get() ); if ( ss->m_Jobs.IsEmpty() == false ) { - Job ** it = ss->m_Jobs.Begin(); - const Job * const * end = ss->m_Jobs.End(); - while ( it != end ) + for ( Job * job : ss->m_Jobs ) { - FLOG_MONITOR( "FINISH_JOB TIMEOUT %s \"%s\" \n", ss->m_RemoteName.Get(), (*it)->GetNode()->GetName().Get() ); - JobQueue::Get().ReturnUnfinishedDistributableJob( *it ); - ++it; + FLOG_MONITOR( "FINISH_JOB TIMEOUT %s \"%s\" \n", ss->m_RemoteName.Get(), job->GetNode()->GetName().Get() ); + JobQueue::Get().ReturnUnfinishedDistributableJob( job ); } ss->m_Jobs.Clear(); } @@ -628,7 +625,7 @@ void Client::ProcessJobResultCommon( const ConnectionInfo * connection, bool isC ss->m_Denylisted = true; // -distverbose message - const size_t workerIndex = (size_t)( ss - m_ServerList.Begin() ); + const size_t workerIndex = m_ServerList.GetIndexOf( ss ); const AString & workerName = m_WorkerList[ workerIndex ]; DIST_INFO( "Remote System Failure!\n" " - Deny listed Worker: %s\n" @@ -664,7 +661,7 @@ void Client::ProcessJobResultCommon( const ConnectionInfo * connection, bool isC } // Record information about worker - const uint32_t workerId = static_cast( ss - m_ServerList.Begin() ); + const uint32_t workerId = static_cast( m_ServerList.GetIndexOf( ss ) ); const int64_t start = receivedResultEndTime - (int64_t)( ( (double)buildTime / 1000 ) * (double)Timer::GetFrequency() ); BuildProfiler::Get().RecordRemote( workerId, ss->m_RemoteName, @@ -930,11 +927,9 @@ const ToolManifest * Client::FindManifest( const ConnectionInfo * connection, ui MutexHolder mh( ss->m_Mutex ); - for ( Job ** it = ss->m_Jobs.Begin(); - it != ss->m_Jobs.End(); - ++it ) + for ( const Job * job : ss->m_Jobs ) { - const Node * n = ( *it )->GetNode()->CastTo< ObjectNode >()->GetCompiler(); + const Node * n = job->GetNode()->CastTo< ObjectNode >()->GetCompiler(); const ToolManifest & m = n->CastTo< CompilerNode >()->GetManifest(); if ( m.GetToolId() == toolId ) { diff --git a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp index bbba846c8..1a5b037a1 100644 --- a/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp +++ b/Code/Tools/FBuild/FBuildCore/WorkerPool/JobQueueRemote.cpp @@ -217,14 +217,12 @@ void JobQueueRemote::CancelJobsWithUserData( void * userData ) // (we can't delete these now, so we let them complete and delete // them upon completion - see FinishedProcessingJob) MutexHolder mh( m_InFlightJobsMutex ); - Job ** it = m_InFlightJobs.Begin(); - while ( it != m_InFlightJobs.End() ) + for ( Job * job : m_InFlightJobs ) { - if ( ( *it )->GetUserData() == userData ) + if ( job->GetUserData() == userData ) { - ( *it )->SetUserData( nullptr ); + job->SetUserData( nullptr ); } - ++it; } } diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestBuildFBuild.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestBuildFBuild.cpp index 3277f63d3..0db8fe20e 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestBuildFBuild.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestBuildFBuild.cpp @@ -87,11 +87,9 @@ void TestBuildFBuild::BuildClean() const // delete files from previous runs Array< AString > files( 1024, true ); FileIO::GetFiles( AStackString<>( "../tmp/Test/BuildFBuild" ), AStackString<>( "*" ), true, &files ); - for ( Array< AString >::Iter it = files.Begin(); - it != files.End(); - it++ ) + for ( const AString & file : files ) { - FileIO::FileDelete( (*it).Get() ); + FileIO::FileDelete( file.Get() ); } // Do a clean build and populate the cache diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp index be1f266e2..9036ecbe9 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp @@ -105,11 +105,9 @@ void TestDistributed::TestHelper( const char * target, uint32_t numRemoteWorkers // clean up anything left over from previous runs Array< AString > files; FileIO::GetFiles( AStackString<>( "../tmp/Test/Distributed" ), AStackString<>( "*.*" ), true, &files ); - const AString * iter = files.Begin(); - const AString * const end = files.End(); - for ( ; iter != end; ++iter ) + for ( const AString & file : files ) { - FileIO::FileDelete( iter->Get() ); + FileIO::FileDelete( file.Get() ); } if ( !shouldFail ) diff --git a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp index 3eeb3ccba..cfc8db46e 100644 --- a/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp +++ b/Code/Tools/FBuild/FBuildWorker/FBuildWorkerOptions.cpp @@ -48,10 +48,8 @@ bool FBuildWorkerOptions::ProcessCommandLine( const AString & commandLine ) commandLine.Tokenize( tokens ); // Check each token - const AString * const end = tokens.End(); - for ( const AString * it = tokens.Begin(); it != end; ++it ) + for ( const AString & token : tokens ) { - const AString & token = *it; #if defined( __WINDOWS__ ) || defined( __OSX__ ) if ( token == "-console" ) { From 4c547b460c37f7cd8d7fcd7918c6cb2e3be35e9c Mon Sep 17 00:00:00 2001 From: ffulin <11589403+ffulin@users.noreply.github.com> Date: Mon, 25 Dec 2023 16:08:28 +1030 Subject: [PATCH 104/129] RemoveDir improvements: - RemoveDir removes dirs as well as files by default - RemoveDir directory berhavior can be controlled with .RemoveDirs and .RemoveRootDir --- .../docs/functions/removedir.html | 20 ++ .../FBuildCore/BFF/Functions/Function.cpp | 3 + .../FBuildCore/BFF/Functions/Function.h | 1 + Code/Tools/FBuild/FBuildCore/Graph/CSNode.cpp | 1 + .../FBuild/FBuildCore/Graph/CopyDirNode.cpp | 1 + .../FBuildCore/Graph/DirectoryListNode.cpp | 73 ++++++-- .../FBuildCore/Graph/DirectoryListNode.h | 4 + .../FBuild/FBuildCore/Graph/ExecNode.cpp | 1 + .../Tools/FBuild/FBuildCore/Graph/NodeGraph.h | 2 +- .../FBuildCore/Graph/ObjectListNode.cpp | 1 + .../FBuild/FBuildCore/Graph/RemoveDirNode.cpp | 75 ++++++-- .../FBuild/FBuildCore/Graph/RemoveDirNode.h | 5 + .../FBuild/FBuildCore/Graph/TestNode.cpp | 1 + .../FBuild/FBuildCore/Graph/UnityNode.cpp | 1 + .../FBuildCore/Graph/VCXProjectNode.cpp | 1 + .../FBuildCore/Graph/XCodeProjectNode.cpp | 1 + .../FBuildTest/Data/TestRemoveDir/fbuild.bff | 25 ++- .../FBuild/FBuildTest/Tests/TestGraph.cpp | 1 + .../FBuild/FBuildTest/Tests/TestRemoveDir.cpp | 174 +++++++++++++++--- .../FBuild/Integration/notepad++markup.xml | 2 +- Code/Tools/FBuild/Integration/usertype.dat | 2 + 21 files changed, 336 insertions(+), 59 deletions(-) diff --git a/Code/Tools/FBuild/Documentation/docs/functions/removedir.html b/Code/Tools/FBuild/Documentation/docs/functions/removedir.html index bb8bf9ba7..f04697367 100644 --- a/Code/Tools/FBuild/Documentation/docs/functions/removedir.html +++ b/Code/Tools/FBuild/Documentation/docs/functions/removedir.html @@ -37,6 +37,8 @@

RemoveDir

// Basic options .RemovePaths // Directories to delete contents of .RemovePathsRecurse // (optional) Recurse into sub-directories? (default: true) + .RemoveDirs // (optional) Remove directories as well as files (default: true) + .RemoveRootDir // (optional) Remove top-level directory (default: true) // Filtering options .RemovePatterns // (optional) Wildcards of contents to delete (default: *) @@ -79,6 +81,24 @@

RemoveDir

Directories are scanned recursively by default. Recursion can be disabled.

Example:
.RemovePathsRecurse = false
+ +

+ +

.RemoveDirs - Bool - (Optional)

+

In addition to deleting files, directories are also deleted by default. + This can be disabled in order to leave empty directories untouched.

+ Example: +
.RemoveDirs = false
+ +

+ +

.RemoveRootDir - Bool - (Optional)

+

Directories and sub-directories are deleted by default, including the top-level directory. + Deletion of the top level directory can be disabled, while leaving deletion of sub-directories + active.

+

NOTE: .RemoveRootDir is ignored when .RemoveDirs is false.

+ Example: +
.RemoveRootDir = false
qeCK{(WFyDO*ZqVSg zp73=$?uVrE)PMUgl;l%>fR5?Dy|=Twpy!^-FIMM;B(9_~OzjJ4A7mzu4;c3F!1Md9 z{JNjh5UYt1BCSOvAWdX5=CZ?WTs&StCYxd?O})8NQ>>QLoLQImC;X6=O!2KuOQ7ix zGnP|NkT2t5FsSGqlTubkHs_d7pwkPg`fhzI-?IK}|Ft|d5g0CEMD|P ztkPbdvHkf`@4P^OI{A&%$BEwkeiaTR(!v~fpeT+*czc3a?ogb~6c)8rW6Dpm? z?#-BI@Qd^?_UFS(PT0tLXc?_Bf?18h8-4^6=!aP+@TSws<@b)@tqP9p$)A7l#s2NUDckT4 z2(UpK24|DmR#IeXeAdiTs&6P6#!Rb4MZe|&I+_tTDvPFN5tlMmotba_ukGa2rtcry0(x`AVOpl|dB9y_B8Ifu&w~TBwQ{ z;`jiI`4H(7l0M)*-gRYsnkcbi$q2Ok6qR4F_hhT`kRda0ydFa}J-V3{!)c z#S3_+d_OvDnxuYP7_f;%90V0>4_a`@ECu zNsmwWhPl}bh#3U|?Swi}Knz1~X4WqwoOc1e((B|!Bm$~dF%?iP03fgXg~vWbdZ$lI zh;+@%KvA>OfMg!0K|#v!2J_X1Kd6c96=fUF#e0#YHA2&limML^6b2x!P( z6=E2AGqZj^P278czN(3FL5Gl~0%9Tvh;az$0WTm0Ljf_<3CPNkC?FL=iGU{c%9l5B%qR%xATQp)#4!xL znOVP(0{ZDq2&h`cR6q*>0C~L#eo;UYBE96(5+c1U=>#<^tw<(5TJ0aQQ0S5u&?>T2 zKuiPyF%AL!ffta1p@5j_1Z3q%6p#v`L_n)p=L)`H;+Rno&^k;Q1;jA)W@h~&3g~Ax zA)sm%QvvmSnFl~#9|ylEAPJE^;nNZ#y(Z}dH7lK#OuT&4KV+fMB`=_LWT}9d2m)dp z0{TP9f-F)n6c96=fUF#e0#YHA2x#yLEdpXjK|o_Woq!mI-ps6DOaUF(gn+75Oa(L^ z0Fc)`U*`lQA=14*Eg{nVlHONW)U32qGVwiG|B!`3m%M;#$Wj3@5d_3I1oS_=fD{Y` z#7rk3D@US$R0t&kn*Z_^0WqT>pr!xj1jI1(W@i0z3g~AyA)sm%Qvodp0Oa)m_{9+< zA<~0BEg{lFlEykHt<;cATo?BbStxYL3#f@K6%Z3aK#W5`t6o3~h5}-y6Off7Q9vq$ z5&@n5iWUJeqadK2_?4xsgAj(^%&hlPK;~`o#Lcc&F%?iP0HA1Kmz0 zF!W|-{V^2KL7=bdW>>423g|=tKmjd)UlfppNFVZP36WluG}b|Br6tKcP*uL}+StxYL3up~l zDj+8Kk3wJ^0{Wa6kb>beW;y{`IT8h=LMRc?|bl-!n2HtTUd>G$hTzsdJg z_v?4VcS`o_d%mCCypVnMZ}nM2mFE6%FaQ`>WHARn7XW*NX%xfQ8(GnZNRLbUfD6;= zX}xfta}rk@gp%y=^PoATw6Xb1lEx-{V-ljVNu=dH7HAKZ=_oNP5Wy*~){gMC8El9_ z=FsA=K!>z{o%X|U5e|M!fXM`pJwkJe`Shg;<|&U^LSUYbm|2;anH7lO0~=a9V%BC9 z^Ajs!*7`EI(Z$f5VjfL*>1I7<34wV&Vy>YwF*7R=!JS#H9Wk?{lu*oHSqZb&C+6wC z*qmZs4EtVqWo>B}95PVrFGxW>&-;Ye&r5jADLDCCpl%n0s`2 zp741vUC$o(m?Z?}6A?2j6Em|S=2$yo)@BrQy%J`vPt4=GVofm5<9JzhW6fih5SUL# z%&bhz%!-&}?TA^MQOsXc3A5HG=H;*gO?BgRSlLlG);(qkk?xt$Ifa#pnOPBYtQ|3H zGm6>#);gI}v_3J{YOy)B8>3H7_}mLXoOC4w=KhG8m5G^I5p%2^F>5o5+5G)+!L0R( zdHSnkbBcK}`2XbdPLEkaq-zl~D-$!bBIZ~-V%BC9^S}<&4Fxdccnb2oGBJDA= zGBGnt$_wULJ7U&m6!SAGVb=P@+;eqoPB9NYBf&i8F-r){;}J6}6Em|S=2$yo)@Bs* zGb>@%`oui`%-Eb_o_|(?dBS6s5SS+;W>zL9mvk6xQ#p7od|1m^jOnU#r| zSrK!r9WiS&iusyKn6*AJPk%#fPBAY&C&7HsW0nw@7b0d>CT3Y$TYEd(097^NEO=m5G^I5p%2^F>5o5+5GZa@sYGX zF^><&<`nb%^AgN!9G&m5G^I5p%2^F>5o5+5Bo>=RXIS#9Vt}Y)&zcep`aM7l6cnW(dsv5i=_j zGqWP*SUY0YW)$=DD)CwC6Z3R4Hm8^uUzA|pNv!aZBn0MK#LUXX%&dqx){dC98O3b= z0k7~`>l5>dZ;#C>=Khfc^MJ=JAuta{%&bhz%!-&}?TA^MQOvt4@mcE=bK}LaImJBr zk_7XR$1EW*HzHf ztcW?*j+nI>#r&d5n6*AJ_q;4NrlGbkJiZj*G?IEK>d!+XFT(Ga7H~r_IS*YFv0R8fvTz=HkU9H9v+0&$ zb{;w;I*9EzhG3i1NnA@~I;{>}2P=eq`s$CW=b=sCn1pC-6ls5>Ih=kF_IuZAQ_*vl4o(pQG19$`pM(4F%dm9=(J} zFGlp?ER|UieXJePYcq;|wi0@+pQG2~$`pM(7X{iSk6uEgmm~Ucs>-a0KGu%twHZY} zR|&n=&(Z6_Wr{wYkOJ+BM=v4Ls}Vgbiyx7pkF_IuZAQ`WtAt+b=jinaGesZIN`dyc zM=v4LCnEZA(#ovJ`&c`o*Jc#`{z~YzevZCYiw&zk6i-cocFm)g5b4tqJu8#<%!=q^ z?TB8RQS|eb&};o1y&j9EypQLnK)deIONeyOYt>T-r?1S4=wt1OUYk+$@2P}d>*wh8 zfHOrOPf~%l7tmzCjv>I^>g%k=$WFAr>j6a zK-54lA<}~peK?zCRzx3bNA%i^qJM8C^jbehug9M$`gqO?v_l@fgh)3c`fy6itcX6= zj_9=+MgP7^=(T>1UJpW3^zpj|JK(k6uEgrz84sn#-(+KGu%twHZbKflBDL zevV!bOjGpnWEN;=J$ea|o{#9mxh}IJ`dB-n*Jc#`2P>i1`Z;<%I!)2XGg_cM=+R4v z^g=`*PI#FW(Z|{my*8uhKU4|5*3Z%FA!>>~p4I~GA&*`{q!%OlaMsJLh(6Yi=(QO| ze|shLT0cjx$Ehj$cy0@{OCG(1NH0h9;nbH|5q+#3(Q7k`{#PrZ*ZMhnJy=cA#}izj zUGeB8M0z!%XJzs4krmO$+7Z1rqv$`pql%}X^>g%kM4O_IXSqOo9MI?~NQm?aq`jXL zPJ)>g(Z|{my*8uhKT-+3*3Z%FVQY#$p6UYanny1o(x)T(a3;*Gh(6Yi=(QO|{~MLi zYyBL(9=oRKVh(6Yi=(QO||A|WIwSJCX4|-Gd@kAJCn;yM{NRLMJ;XIjH z5q+#3(Q7k`{x>V3*ZMhnJ+e*F$FpIe9rNfVM0z};4=2mais)nQh+dmf^hYY8*ZMhn zJ-kiP$5UdUo$%--M0yfwf9E)yF*7TokF_IuZAQ_5vJ!f&pQG1f;1qp4F9zBvk6uEg zrz84s+RUtoKGu%twHZbKsY>XzevV!bbW`;4YH(`Z%QnGO``if>#t*1%(O>#)Vmw)-?v!$ zGREuSvtQiP!yglW)Y{aYeBYtG!cjY}I`DBTk-K>et}iNts^VhGMN#KEh+gCqMNi49 z=vCe#R+0C$xx9NWp&-+fiz|E0$eYAg+m3}0L#``C>>(@X@T(4BxbN* zTAGR^=)u3BW%{Olu+YE2@0hgix4}E@i~bg=$7Zv(griCvsnEfa!6Y1r93}6Rqsd^3 zzh@rTUw(%3`~C~EQ8kGIIhRM7v)}QZAGc591I@V`n4t*4rkz5-he+2XeZYs<`$0NL zK;JCH& zXIeUyGJ}sMdrLU86Qe;n!xAj1w-h30G)r-vu;LQV3}keb$lp~UqOZXYKO6shYh~fg zRA21=zWO0e`7b{N70_ZaU$AQhc0>ztKo5vwCnUK|Aeq;%(5#-c!n}QW=~OIxqzB)_ zbfWQJxJn@ZbBA*4S}ubMq!$1w-UrX1r|%s+w>!#40GC&F2(vGqECj$+<%{?f?#nPng6gKYk6NYP+~0RWR+ z(sTbVd%R|M6}eD4(Dl;*){kP03lnX-FE9O(6gSL_oxi;F$5t{Md}{Get#p<|i&0v% zL(SixS?e!TGj%d&VkmeZ>ZXi`99Zr6sYK1yM*r5@D865(GiZ^_2`ib_ZzocLoLx)1 zk$l|X3s%bV*%L(u#S1!EqV}SsQSGVTiQ21^qx$cZn(KvTuKzMHGX4Mm*?SxKO0TM3 zIMdFwDJ>)|AcG(V-=GoD2B`OrK;h&%=;3Fs#B0@2K5i7n@sl7k8nt42Goz`QfdfSD zfCAxr_>#;(OQw`^3Vln86@wyoeD!L4FM2ah)o9ThKcM&jUwiMh_kNz|oUD_{nVFX4 z*Umm`f2_6kT5GSp_Q&%)dvg2th7LQ2p;rzCRheqE(aPYz)PXEiS&CM4PFMU-Ry09b zh~~tjvA3eEkrTVHz;K`VgQrXKFK0gR=k;@gh1VPcrjwU(9PpPj0A@2rhBsw8J^6Tf z5A_39Ztu>h+`V`3mmqTQOlmqDoNN`=fp8fDKs9mjl#b5q`a$YUax|3fesIYMh=Yq`r)qG;^}{l3)tNw8Kl}`)gyjD3 z3M%BYOp`e}!duNUrKv?{ni`ENI7V0@DL585flFbU8O&%LB~r}~QH9lvo@ABi9ZDXK zuZ-s#t#~k*c#b(KsF}O(E}gtq#ze@abu>hBNmUL)$$ZLEefCZlT`Hcexpwj?R_2=A z!wdX@4kvh#pQ3uI71b>nE6{i%wR+G`qc`YBsiL0d3RRBMr(3DYR2_{2@;e6kl|E|4 zOlLms9ahh<;-6oEvhxTVGyva#saj{F(*Ie|+ZxTGyO7NppckGZ;_c&e^ z7D6OsZx?qNv*F904n1$sT`jDKm_Jn48(3nmtS|$Kt%X`A90PDzpu~B#UULAVoLVwtH3C~Ncc;pMq z+MZUmKRCWdSieU?TBZ!Y%u47aaUWCM86`KMv@k29tQW*QlIF}@^qI;NGn>d9h5V%j zBvh`10mn*px4`~MBl~@af5MLyB?l^0Zh`l{J9_@lGREir);13>wkOTff!O%ESki# zMW49_ZVEor`OC%SYXYAkp8G()AH0GJxz}7`PVi;L{ot5C%zagC5Mv5c8puI*AqTs3 zql074LB&ugWYKnJgrepEf#hfqq$P?-9mQ*`a>rI_sWzGq3jhxkFca;zfyTflB;SR#GByZC_7@@VHXb z0jOC@=N>B^9NeTn7J6HRkV|tEGo2FM9Cfc1|2&1R{L$i@)GXBGQklE>^MQUA%J-Vk zVk~*BYtnwy7lkDXLL0EJpGng0f6mI@+sK|0)ApaYwr^(LbE}&_wl)@j6J=B7{P^bN zFIe%3ztPS}9H7_*(pjP+3|OLCVi>RtFoXfRUYf$-aqFdh;3?V3;mJfk^G`_CFspHQ z4sk`WAx5O3h`;uUX<_u>oB{-70t>mZygt3&&&lAgtn#=kw&i`Np88MUb7|kLJA1GH z5CCHOYb&N(D=^)?v-fv3wt;68)^#VUAs{APq^@_|Czhocf;K4)`o`F1En3oYGL6r9 zlvoXDWEx|hB$wl6OvIgLVq8e3Mm}jFNGVb^`hT)YLyUgCH*_paGm&8}Q{4Yz#f?RR zY99UC=gr7j2zxq?94-!|1fnEN(J9Tk^HBe4rTA*L9s-tNglGt3_DV|2v^QU|;+7>( z#GUaeBT}_Mg3g2zD^P-$aM2zPYT`hW@@yAdHnun#r8AV*egGwR&P-Rs32T=X|9rtp zA)eI2RduuXdUhQZ7^66Z=wIkcO8#X9QycwD?bIqHocI^Mlpyynr==?Pci19C|-iRHm6ZAQJN}a1{q@DPI!+mRi0BFX@ar*F3WL zAB#>5G_j0ARnob$>*8pO^cINEKhlsr7&W678k2T()1#c z7LbByQECSHNR3QBwezOMr7D(}JFbXw8$W^hDJWYOM zq_+YvM0)NDj;<$b2xZ-R(s=*IFRw~F$)kfc2{geN9x`Njg-x#k-A+u`o`?x+3_W-< z;@Y^YBMKu9^QW&bG3x!`$CHRK#p8nOAF-mrMKr&6TSoGuRwS~cs_hv9OLz6?zoA4l zEB5#&tyFGHdYs~MJ0L%6MS~t=nF=cqO-K+3AP*{0V_0iAiT<1w|2#FV{E68*L#Z&^ zJv7E>wzSRMY|Hn@8f+rx$1eg$$$Dj;1=;g{$g89g1hpibq7Ie8?1#8W*lR2Ro9{)AA#EV&8G}aAfG{sr_Jm`o7w! z0qDYyb1p%rZyX%zIdFA-R(Aq1oeiJ8qW)lh^j$T@8wMosyR*bn zNjBe1-={bRvYI#FFxY?d=wL6|><7`IZwJj^xB|ZhZr(X~_Ht(7$L!9*`t)7>SyLh4 z=d+QE7FcU=&e?)9SPv0cC2;*ELxFvB!G`u zv34YY+wEBG0^svtCa>I)0PeJ6?PR+Dq!npLtp2}NtR3mvC7Hey%H6|FA1q|HzKjZdjkUB7J`cEB`;UBkf?5 z;(uDPcH*&DtVlcQ|6g_k=@+|!^snqlt2I@*;`tjZ)>eL{66@b~1?%5+1?%5;1?yT@ zu>M0=u>PYN>p8E^{@F^Zv(=gTZB;yfDW9~u1DMUN&mOS$sMP?fT%YyuVBd0})QJP# zVMW?eWOAt$Yexdu*%hq2x`Oqxu3){QD_9?9$7+vEuCyZU$gm!6#oCc!J)$dE`?`X) zzbja;?h4jx>{#uF^(ZUSPC~v%+mTiWQk5S2F;=V{S?lhuU>&eywd?AMR-~QiYS50f z16@7IinWv3^;@k-JL&(m-9S3r4Wv)ABdyj{I&9p zcLnQnx`Oq&W~}GDJoyeQsm_)sd+lhG<%zCGhTGmbaF^8d5chknv|5cAH@g+E{+F&` z{XRR^HuF${)f=oxTLq)eWfZXfkQHkyu~uSzQCG0OxGPv+(iN=#wJTVEq$^l|v@2L& zYR8&fZcWdVJMs_v?c}QG=}Q0blXk53k^ECuq^(ABC96MU#oEzG{@HWITFL9nyMlG1 zD_CFA6|Ar93f2Q%!TJkb!TO6`!TL*G!TQTx!TKv*!8+X)tcUDa?T+xU6=_F~@Yk(a zTRFl?m-QQVtod{n??(;Gx75z%c?|RS&27(PoHIY>FE^hje%@tOQagUS{rp>8|G0~I z+s~gpIG(+0^@x+#r*nJZAuV_GCja{Ko9^SIuI~BcAH(5V^89gA3c`bx;Wf$Om@MS$ zQuvwHqd9pG+K^ZojBIawS62hiIVK>UnB2>UV2^n|{Oh~$bT4zlSAGH?cy?p2f7|hM ztem`=Pqy!?AI0;+Xd)an@FL820{CR#rG-bx(^`Bl(f7ssnR<4ZAiACy__WdUtlWI) z?DrZ`KHrRzed9ezH>T0IB>+#C=6lg|NY{OU$G>_G@QM`re8B63@unw90n1rwfTx@C z!o^#2jj}w2oJY6Ye`C;RV{eQh6J>9K3)xZol*I>?7qpQ*8Etyx>{( zG48B7rSS~_#3GP;DfB!mrP*ZZI5PR(H&-|(aE3{(`TN0#s)=%VEum4q#;Dwt*aOVAZf7O{6`d5vrLEPVWHC`GE z)H2k5y7_e2^^KR`U+;Tz^Xy~sqH$h}5;d zzwd{e_}{%Et#4@gWzX+>)$+^geXsgV`oczeEPMw1dzBpx?|GAF!ks@TzER<(yB5G* zh4<3?-8bDgFCyXhcLwk)@m~v4;Ji3zv-<34cs3ZIyal@Ecp2rx*?gtN0G*M`R7?1I zL1TFQ1y9q5#x#RtYzG5m>datz{W2QCFIW-crOb-Zx1F@MmDg6fo|J_;4=hEYZuxw0 zlawY3hu<1-=e)L4%+fuvS%`b|3%LnJ6jiB)w<}imPLy@Nr?Ys{gX+aoH;q=QsZ>ypB}4twJUWI-~B=@W>b{T zPkcR_)9*m8-vt(A?@MC;$wK^22j=DdQq8A;Cj_2zsd6YCiS2-7w|;Zl2_;dw7O5)D zm-0VM{5XCVeNh0Tsf`utG}kqePmyufmwGxW86jt>hV- zRK_rU%mDzY^35&`^4E7R4$Z?2>-&2kZ*aS{A7(F z{e)aSyx`&E9$sP!`AZ73y5S>PzTM$X9Ul4_wEaifTaNnUChAi>9Qek0*6<`BB zaUel+%w0v7dfWm>{9KwXYY)-;p+Q%Ow?PBWFsX(9wN*GWp(!UMvd?=Sros?)3D*q7R~{Q6HrKrzs-k z{4$jLpJf(g{%29D!u_kDAUz=heM<`Xm^7Pn>CwQQupQf_PNKLIlZg zk6DFOdg}14ivZkKO>|D;bl|K=h(l4F2H@a~R+$guk`(1@DeRroI`W)zl2Y%SLQR&^ zIT4P#HLg@h(iM=sUZnhUyETOTZjJoJZVl69w`KsK^bIeMOkA6DX2)qnX*N5zAV!2; z8!5yG4wzyjC?+-cM_Q#U&P>~r$RrsDWu~j_PyKp6PepSuUow^8I%2*4I(_brXucm-^sjHY#yRL4t@e~7`0Dnr1NbPn^RIkYdU75)%z|ux zpkJ!uTU}R$wFj4U*!q(FAf|v%3miURob3<9c438y#ZeZpkB3!_gzsAB7F{dE%=J`g zLUR0)FONi|5L=J(WnWZMOuFcLjnz8DXtlk*L4A6C2b~trpG_-d?0P3`;6r}W{B_pm z@`+pJBA18Khb}US^(Wu(O3Ix4*1Y_*3FWu$!AKFrCXiPY7Akx3>B`Q2%HSe*3VYi%v^)7B!t*x4w*VSMK#%1^@SewiE z)#f5W?Jl6;7>lHAQq=7cG5Qp6FX7mB9o^5deuv0k?8`Ea z>9uT_WuPW0c;Dk;NdedLwrj9QZCm$sB}Lsa4@(MoA7IKs=tB;I3vyt7kV6y9P!5V% zOVF1oT`?BBjE-84Q@UWzSC-b(o-lXl6uPZ6)&Ujc- zzzx8ZgV2W@1Q+DM{2+&ZP6d#IA{IF)i)hUn(blOg{rswV;jIrx2Z} zIw$@77b}%>(GS0sx342YQ9se*sECrqX{O5UB^?pc_KC#}hiN`)2djjTsf2#4BSP9f zAv6&d87U#nR6;-15g~1#5aP<9Bs$GhLK7Vk()J0VwcZLsnyG}$xBkWIw0*^3l~EN3 zE5~DTu)^s&f4kUtl;^Jglofi_;ZwVz)nwGpqe4DvXPAQ-dEk+uDW>uPspeC_Ljupa z^k}w{)pob_i*S?WW`h?;xX$DsY3QoAfv;A61O+=1$u1|hg6#~I^WAt_1N9KIBTHVD zKMGW=%~#SO&X5%?vW5c!lG65ApUTd|tnlEyQ@f?cop_r_hG2F{EK7GnAdg`YnnQ+g zgP_z={U5PXd!4(!yAi=XIYELs!J}4!ugN-xap`0!*r1h1_2ya`1}tS#bAN~>8>$s% zsa9CKomRAc*v~b87R75NBB;Et9T~|RsNC(j8)N~sv7Pec$ z5OpF`Au&sZ#MjV`0j$ZCPjWqk7&&r{dVXQY zwLAwLYq#TC+lK*X{rhRKSyyJjtOK>bFNH8a8{vd}8IHe_8 zQOQFq*Fo;7R>o8h)C#|mTF?r!)RtJgomRAc7;tx_)Y{3(Yb7G6(8|7CNCa~seZopj z2At9oA*moBq&ae$nC^m#g+f}z7H>gF%u*q-b~_1JZw=Kc_IYELs!OvO=is>pX(TYkQTKN)$oNA@NFX(|T3( z*ZjnQH{FP!LMs!wkO<~NGQYh4buM0LOKlS9^GnN!8ki|G;pe%4Fu&OU)wy^Hspz4W z8(bi!ettp~!J5afE~$lCsutF6rxtA=)WR<4&@>E|h@e6(Q@L6Q=4v@(r6wLlX^HKs zARwfVpwE$Q#6T(*3aK{)!xHD99->cVsgPK^oshJ>pTfVC5^E1E&@dR440)epJZU59?n`Iga{Ay}*Zm znShToypADp244u8upXfxu@x-`MhdzIW3Y=X*~!^~Y(0_)HU}4(qHR-P9P=Vg?Bn5B zZ!94TD0gK~yVVuUV26`UAv!&qU>P&r5$`S`zwt7m@9+z776B3y*cRd6*?#ax z;DII>AQTN8UTgehA-FTZ^o4d-=2CKbLtv9gE{a#k`uLJoSBV&DsUkU|goy!FvkaBqD3G(2P ze>Suhi2G+l6;}E19S|zvSsXcW*~lVS z3b*ke3YTMEtaju?uX`BMty$BetRA(w8wYTu8IPHQ$-6~dWVH3}mJLxo)r^WNS3y&1 z!Q%`lci?}O76r4D6!4nBSOy_=LzsAClUx{X#{kW0h|l$j;_(>TC(6=XRk~B-#y;ra zqM8P-N(JPK#(Y^mvz;=YXIn+}cC$-V_t6kR zRfKO6V#V-{L0Tbv3-9;4>AqpCs`3DBh&f^IS1K^SUapbh;v@wu7pDMA3V1imL&GFs zqI;KXuc#|2;31DoQozFk&p8|z%Y{>1d%tH>Qos`)mK5+L;n-!ppTV|MzOJNzXFMz^;D*4s=pZSgJEWn@)(ao~ zZq0kps^#h~>jet3eay2dDd2ezOA7cn;n-!piov!EzOJNz7d`i0wds-6p;_o&}G{-jweka6mXq3zi9U^cfsk(+_al%XtwSdl@yHjh|BXS z;NH~XVQmlsBX*kmB=mWb0^aRmNdXTCj5}SDBEUl$I=uiUsAokedREqel{KEk)u-$9 zq}907PQ!(W$BzDwG)lXc3#c&q+Qp&i8c@+Hnx|XQ;4E3^zcQL9-{!UeF!w_= zG7QRZ9ZHHWFRTu=$jJ=oh9F!d6L12tM44l({YANp=9=I=C%xhBu=7OpBPT9^p8R;d>Ao%34|W!BM;GH7Dz&OM!=aN~G;PrQ$U{p{ z@&4WogLv8N=wR;;)^ECR0Diz-C%F;bO`Pa33o<9#FF8jB_aJql26qrg6=>hL0SxhnHCGW~0wSnS`g;;Le*U>;xdg!I7oFvgG)AX`Ds$pO;fFQIYriX3cP zdiDsJC%=xr>Gholo||X6LwcEFMiytRCVAYQTt&=ab`4~LV*sNSW3o94^7(7tPlyojFobu zN>+|^#tQb~%dGU~tPsptDL1NQ=)4GB9Ht3Z)i?Tc#mNmk9~1_EaGk%JiQ6BGs%^*35k zzpJg$jFV1`jVA!6Mx*w^Yv;~hcINiNQsvI+&VO3md4xOwp>puu-?-pKm*+D4;~>6m1Z?jwcSZ zZT~C7v5WrKe6b`SjIfst{qGN0ZDnixZ;b<4#wRyhlJPmlf-D~5AkW9=SBw9pk1$`U z$yHwTzvk;SxRQkU03u%%7YWm$&n>wcX1W^gO!mjE=F{HQP}hm8p_K$zBT=>b8}UhE zKNoR4&HX&nq}N_|=I`EaWjswqh(FqOnrRBpR1`wx%h}? zB)JG`gK60sf)ix`b|W;Ubv$*_ip z{{t(<3pxK^koiCd>aYw94L)NOh}d8c%eom`(%t`0tByA5?*H71zYFd@?%!?42aMy- zc`-tR`8%sZH)$|`Z^hpQgVFflBHs`2Z+X51Ya5L$+ntQzdG$6EID>^G%x2&7Fk4R` zRX*GMv)w%(^9t^>D>yFM_j9OkxIrtozQV$@o$b6U*$u$q>~b`v_96KOM`!u^%}s;6 z|G#~HCMWAIK+sW(b+$K&C)iCwTUYF&$nIENFj4*$V_wJmgJQk(G^=Pfxn6p@6+dRC zHtyE6UfMqY>gxRiKbW3t)#xWPCus4nH7n}#;$LgB%%FeWJ|Ao0U)_skzQ?LI@gZ?< z3noFATH5_$=BjM=y4|lASn;>_Au?Y^_f7RyxbL|Bd2qzDKWY`u*2l9DpmBd{KdN37 z`^V0#V0bQgx*&T9-jBY*uDBSd>LEhdp!#_$mMz<$dZm@(s}uh(M`GMia7SYNxs?bdu%%52CcfnvZ9({1^ z2i#{B)z@0u*oDgTNEy6WdaalZ99t7ta4-g>+hKW3(| z2Bpl(l2K>{|s5Cit9 zu_@XA=b|mKhY9%#6)>ROy`OPRs=6HanLGVGM2*%$U-4>9wt`<62(MDYS4?J*CtoqC zad|wh+IF8@P*}zC6RI7p~nfh>j$@jhwh@WD$(^=vQVa73rpY zMz*?0cAk_GS^hiG5cgPk*$DFet4LEh@pB%m!m1O!YCeH8%~C1-qP2S;Q&}dM70oYO z(VT)z($DUs0;$XwX(-p@z1H+h;%51ct1KasNg|++dv$CqT zk)l;JAGD$o-v={KcvB<`RwN%AIdMtBU8=c?C&j$mqu=pLiY^;j+{q&TQ%{XtM6c2j zNCob^by}P3&j z?tjOb6Q=cYRGAMo?~dVIyCu-cYsGd$U{^otzc-ykWl8No9>$~JD`xxEBwh=g z1rNBKRZzA5tyQfb%-R#s8DRqwe{Uu6z7mPpee8jpM(&y?xmr@huuxc&zL5BG%Al0e+G(F=@ zg|7k!2rb~%Pk zir{%Or<@e+D!6hj5+9<>3Z&}bErDQXs!o;5*O~i6b&cU;vrb(SDFX_sNnANAvGPRt z2rgXtoO%n*RTjRb0~X->3dj}BuoX?F;W9hsm-=s@;WYAAMa3lwb%E> z@=fx0Rf=Ckr(JcUCXrp@@1#tp13+Q&shFV0Ieo?nniFu<1nt*H1CR0Y*ZH{#Cz#C7 z!>EwY&%2ou*Fgix@7C+1*}YgfV^5%7$FE`j8so1${IxH^us``d!5VU9az@N^@8AUA zztoUHrxX8i0p0EaizOPx&c3987p1L-{UiDNvWq7YO<8{2=^U&K1~$(T&epG2`j*! zdr;&D3n0D*#X+MOx%@Q>c|jfOmb8AXfOlmJGOeRnNI(G{FfX?bm}b_Voo;?jfoobSUt$-#r1)uVu3!d3z&BMoEMulS zFFj=Bx$kh@hiuyKunXuf1^l`Z)4|)ap5*EhC=ox#NQ?m>fZebxr)3w&&?b+g!$hfN zs*u-MSvr}u=Zh;6!dV%$vg$>utTA)VXDS!WTt_Cq5G5_ZYPk}|5-Zi8btexaUu$Hz z@9u z$d!?9(nE?%Yp5)C5KrWOpM>+v!F6a(|-^mHE8rFNPT}?-N1O8iVmzL2Z zQ?yS3_X>>9oXJF;@DN9OKZ-!tjdTM1<7nn@jPz690+kNSnCV8k^+n@xq^leXBmD!` z-pfc=EV;S_isGNW*Q?4&2*W}`mKYN;=qbiTdfrhJ&GC7%I=pof35iOx?;Grqsy-5t zsaJpJkr3wPQuGres66I5Dsjc$LELfXKhWnCbmIUYzRV!Qg%eWb!wFZ-2SNxZ2=^V5 zh!<{yP<>Wrf(C!mHj#>mM1E_~go79@uud_Y(E4cwFt4$Mq9Aq!)Y9*<>bjY=7v~4h zov|!-5Apk0Vx8q+y9Zg)2_^a&D^ZChlxY!(cKA%ghY`bB7~vMOuKT*=gd@np2Zd6n zqM3>Gl05hiA3i8SIu!zxLIY;zROOtC!rJT8)?TMhWg@JnDcoM#Hta@H!2JTxxisY= zIu*_opzE9p0shf(@Hd>wnpdLIVHq=>Q~6UP&(W!9znuff#GVN5$lx&$iI%>kP?-2{ zz{&!B(gKy^7mYj}lv5FH?`D0an&GB!2PjMsj5Uxs;JWp;qt7nyofNBiPBW9_572-e`BQRL=;<>y|lQ~=%wk1qnB3yJb@Y1okufXPyF)~ z3o`#aE>$sDP~gs^+n+(#kMdL6(91ablnyY3)iD0u+UsL}T{6g`jC9zO!hP4!k#P3mZ~McEzhESzH0vzC|L z_QYQn!G;{ggkY*SMRgFN>!YotoKyl%FX%qlDHG&5d>y%`MzFyXn4*!`_Ch zB57}LPqCu8@b>mhE34;jZzEPxUAMQzP3*@pWKVX@29n=}D3rHXI9;HDc(?j2PrgdQ9C{47zgr1!9yyn{^EzNLQO5UjON8wG@_e$CeNF^gxJTt*QFUv8~GW{ z{I1meDui=CDi^wFC(1GlRf4a)Dv-(^?)&e&`(E?uKLj`4U{9fzcogoJnB_*mm4*%=sT&j*xS#BlOX(O+?X)8%lowgR$X{Q9z?g&Kr z>X6CJsalsS+8a*EGySeNI*y>r=OB+$at&z@(hN#+AeRgzk$mtu|!)LFke;Ns@!Y2Lk1yngYIN<6`-&IQ@2X}|k z$E(x#ZQq9$A&vU|4tTU$B031M=7gx?wAV~;IeU3KDkW{`^8+{Cw=Axb`#bl+S10R- z6&7Uchb5^R&e~vZ{#2%TVaVSl8B^cy2CTw5V4q0~3i~B(V`!~~=suuTG}l|vV8Hs- z5B!XDM!|Xxj^&`toU!eAUb;UACvtF7Ku-`$*crlD!b%F5zEt7qW{u0-W5{zmT2eag z4&*{jmhym%aNnU{fjrpWmCBCRyr9IrEv82kkrDTCX5t&SOnbPzc!5=X_b@hMGqF&J ze|o)-TkB(11fR$dXb|O3IXbx3o$|pxErKZ+8M67y4*|bv<7b9yuPD^6@yvf^0kvOVvPd z3&!J*8M}Wloj|qoIv#)Aibf_-ZQova(%C3Hib;hRpNG2Xz7FKKXRGD+N^8ek%a7yf zfE5ic%V2P$7tZ2g;gnJy6`}?+T@7dD2d(($Gw0D>KO)SFTtCy+jgTW( ze>e2QXJF3pd8c$aObgPBZLlC5gy@;UsE4?~VI4;M6P(SW6Ai&Zf>Pbg8?9o3o6$2F z;*UQ6z~fpMVIZ0>>%S#IqrFu|e|s~G=4diT3q@emeEUCTR289&zIii@=4diT3q@e` zEmjI(G&Pm78L1-MyQKk|b976z?sVGF2fW<;yQtUC^#0dr9HS&rafA2wvRQb7Icl-s zrbSc?%BTn*y$2?ETh9Pn9T}_#du(Xdq-Jq@X;`|hS$-cQz^b|vR>7vOZJ~tbq!o?a z(_~LF_LQAltaxxwv#)TcmLirRD)%uV{`jCnQdGRZt3~uClxjL8LqEjCW!TQNJ{%T+|B2Yh|B6y+|%~pzFtNc|b&v10t zU)6Dbj8W>=7dI5{E3hDotj4A4#`_ytWR-65iPE&x1qBL@<)!;_kar6bRQq#~!DoGT zPC_{v^b|N1AJ5F16c#dtc(JCiP|OrYikZSrF;kc-W;)hV>N>@PFEmO#ZWePgNGpunUJa7$J=TGjg3?Mz*mrUKvAVEo$WhM9`E5X;~9ZNsKs>oh! zMfTy?^*S&?WOcA9B12!OlIf)#=na3xs#u$2zduxDui2_TyIz_wKFyj`aI%OT2B>p} z8q9JpDcm?Xe2RDK5`$S`L1r*bsT%xi1A7QF=r1pA%rRB9r58q%%z5BnMMZu(U{3Co zl1Y-0TxPOLA=6#bq-J>qg%f{`HS|7;9RINdR}uz@9mgNMA-pKHaeV|)W`?o@>M+@N zBEaA_3o!EM15Gf(A0ECNM0bvy0*#)%vrTrPM2}Uj8aaj0*yHb4x2kp7$SJOc{Ob)` z)#@|hy2MoLa#O9|4QqL6OgAr|ZH~xG4q8d$<7eXx#?a2_&zigA51K5AG=nb05)vY{~rk zdS^#^ikWh$nW-O)-P@jDMV++s+=weD>WhDJ?S(0rC)Zw?TzA2yFRICPdwxAKn9aYb z@2r{`3y!m@O2KPZRY_fDvHXI~uQH9w`r}AGzfN#M%;r~mSzXxi-tnw|UV~L-lvm`y zi~A=yFdzRJ{V39Rgn5}-;JoanZa&O`;DUfdhZ0#~73O8h!@L~!T9rOSw(Ee>&&&KN z^D=)x!jc@3q%bd+FqP-!$||`_tE^ITX_d7K7FN+Wa#S#~7PU(2g%U+Ci|OW0iRLJm z!t7jX;ua3f7S&8o_U_=qVYDOPrp+Zh43DFfo{twpi?cgTG@sp*^}#rvo^|ozRan#{ z>jSmfGT zgkZ{Gdk>|Bp}R-qJtws}t_jC+&BEfi_6@`*p@B-HkqOPMlMrIjAEniNpM(1dLi;&$ zpGydw(9CZglGQQUl5R>stD>riV#nCu#LfW*J9q1!MczYjA`9;e{ycnJV&%MjEAe&ccc8WP6*4mk^$)9I^x`1VD434=^c*rQFj!1y_AHHD9PB(?hN=-{3l z$!~Ou1hd-cU_D22vl$5oVgt3MJ!}dCi`#Vh1~JCrY1F~^!)tp5b$ag;r90srrPST@ zJTkZ(_JI7`B^B1))ZenQnBEa-;(Nb=Q@m75zW$pKQ{->7B9}`%%3OGyV0v9b>))|r zk@vNvyd^xzIMqv?sr9&ru9W9_D>b~kJgT85%olW)-v>a{z+7r}0E6Di%blPqtW=L% zsitb^Pu|>zw`2gyh$(XO3vWaX%3M%`8=fDsatOI#J1$`@?LW(+tO%*qSNKkGbJ2Iw zQ3l@`_YW5r@D!s+SMQS!=KiyZ3c3HBXAb<3dr(fso4ot3Fy8Po9OzLO|6>qAF#B+& z52Ms&LGND#;0C9RT@22`@Kab-bGKEh&^z9oAhi>`if74+XVFPYkDS~CJk#qt8F`pr zTytCb+9-uAxXYJKwa5)Z6^f`o2#`B=!_1ji=K_)54?sl9T$VNwtvP|ap`@^i;`3Hf zWFjr&`GOTsTanUXs|mH+bf-wu`=%5OYj8^(kPn zFT|NDKSzr}&2dx(Jr~pv;2&cb{zg#KASwu3@w1GX&T*bK^4fQJ%*Qps;}pz=&k^$Z zQIk1+lv7+AM1N~WL>*Lc@()%7pJ;*LAI%7~A6TsPhT%_tNGLONHIRR{BFA0R%iO7r zeolo(96cEj#V@UB5tk$pfc-3>5sp?cf&;gx!A!w~N6_hC&2R9TXRp$}nuoHCw>XqJ zQgtZn_^sh^)s0`y8_D<`V?j24Y1A9flEk&@ zp1Pus#kKGf7+p^t$k$UnTxn*bWCcy4i)=MYu))`IJ*C^S4e7gr?n4v`%H^f7sf7qE zuk5xxv@$PS=dVpKyIhr2}WNEfLqg<$k=<+7B1&;Z;XH zxA7m2XPUAQk*SE5a~uE3iljVjhJ@6=mSfUN8+%hxhWn+9hXy5uv?fw`*E|_O?j&-Z z45i%5Fw|u!Wk~qI_fv+hM4C#TNFNU~lN1)vOkuTvCFD8_kaBMU zp)N~l0fe0exDv5|WkHB_71G=Sj#(M@7ND}0C^G81gUDrt%*4orb7B&?6yo{K-#_9I zV_$XA=wGN4R_<_vh)_iu=!@<|KKQO7@o>*y;&m^jrcbZD9`sPZ$LsFyjx6F#0BWxg zf*37D6Vwi;hyf6x)Fi@5cg3=_*_)Yg>H2Nl&vOUad>;~nlXO3ThnHET=gxaKS`SI2 z2k&|aeT>^s`6h-MGb4-8In7*7`tK)yOb$Fum8uk+L?p#bw&OIC)b!yJZwfmy&ao@3 zj{QScHUzEWnWyJ4Y5F)fgG6 zY|EcI>J|KWPMD} zSoOad|NKu@)FamN?sVju4<@+*sO+`0R}!>1My$|kD>_whZ3w}>O} z;xdU}(X1a0IzVU|erllqGpuZxr9YLQ7AhwH7X6>K;=!i9UoZq-7*tHg^@%)-n`W_m zAt~4OM!q6TS8IL~01Y*l(G-CU8#76|ka=XVS2qP%r1!WXL8Y#lTzie7ekcof=#ag@ zTk5_^l09FY-KfrPR%I(2!&oLbt4LUjdHx1II-y+(46$QdUh(2!=g8np=vk_=g^!2^I|{yGv>cnwe;r&tt5LM*w10VCxpRShPRX7q-TM+qFhdW_-^i1>6uALruDa z&Bzu_qo5jL2jBHMoUt|JP8xV}N-&1|$~8PL9kx(6zDT^=Tg z=(56|N3o@ftRJ9u@R;wQq(C$;Fgk`*DmkwycmI&ZC6aue_&5Om$=yHv#@)YhBTi}Prtq!=)uEPXw?A&ge(=L& zizbnDd&&1uv)S$CVy5f%Y2R(v1}4X9skVrpm3+7FFfvxr?Og8y={ON6AAb2b&If$z z8}rF6nI#-OOqe1XM-N*C5z=8LW$6U26MOmP4=a>K+~ZmUSPpn}Jw7h}iu<9n2dgwlunMEmomwDAyd0rJ{0(Zm4)XAF@oY-#TA=)5j8o1((#DM zD;ulna1O)0;{0WdZXx;`I+Qm4hJlk48JAdH?>fjxY1_r>j4#JedoO#@W z65rvf>at+Ye}^Giv!wXAb`hi0S?&^9xOjqXf{23{)ZVYKA8wE&ekI8a9pRM6rxsOHW%hP=z@E~@9T3vxh1kbg+c$GAfr9tziQ@5A6dx~-0F_F41SCa8 zGmk~?XcS5lH?8?%*eWc6Hf7jNY9|^tEmsX&=g%n|H`1PmPjSmPnLisW$n1GSs)m_c z_!@xpoExNo6(X=N8H;$kpNKihO@=`ushL}L+ckP%g!3PX)iK6u-saLEU0di43|~-J&)h`uje+&fHClGf2+vAy9H5v znq~Mq`$#UX3DqBv<%Ima>2}Ft)4u7?qZ$<)>}O38-9{udH08p@+q-!g4@3r#^3M?= zaZ(urlaxEpV1c)x_G$5^V=5krCU;!lc zQls4iR#wTh3@%>r43P4nnaKc}3>8V86-ZriB^5hwoHIha#{)4bTFD@F0He@Fl#W(n z`wZ9(mm*!H#)bg!&uNU1*O>go>qMFEdgar&u9( zL+M3*kGUf<*=Hw@sW4h~=kD>|+^go*+6Z+$*EcE%K zihhM~bHS^6rDkqwy6q@s>h=W6=at;`rY<}2=4huzT3)WntPp1z&L=v{xVw(oeS~qj z@*(RrObN++Gr@vvzHyIsgn1uaSnsd6sDCWXO&Sf&$RuQlhSEy_iiWYx5SgSUB9)Ib zzS@dXKF-Ky&fI}(*yCa$a_|_V`9JlyS8Qf6)TJOOZ#kRzs|YS zey_{p{~ktLnf7_Df*E(j^uHEA(HoNvwKDl+4 z7#K_lDdNv2oE0DB&F*K&mm(+8U%{CO+%56^9Wt$;1pd%U;Dhv`+8)+5C&I;aq(o=u z%9)s{r6L-fE;iuS2s20VR#vw;s`ybN&7O59ht(IXOun{o{*`d&cO3r2it3gPQT=~g zhw4wa4%MG^f+`$_M-xR!u^zg$Jt1-3K&L(Jc{aP4jdp)Mpx7?i zhShuuct&6xi!+4}tVvDrMX?=_3}Y`S6lnz9A0la~KL%ljDtI#%%1B*vxxOR7OvOfE zCPY*X-Eh(@mUA~Q;8oHKSS?AW`cYJjdGQ5l2`X*egAHBR7!qiBiAW0gn3ss8DX-GW z*fh?)1CL11&3kl`0^MTh~< zk^)}x=p-$ni%lcCsc_AJ=$1V?Ndces=p-$ni%lcC1-++0Rga^+c74SomlViXJuE5U zHNt2#kv3ZQl_W)_9vK!s1>EalY0TG(j>6e9liMJ)6`CoTBIanOF3epEXZUWe4s_jU z2~eev)0WfjAa1YK;hX@vTau+dR*T>K1k>wr;td z6F;vtHBd}hs%5*HYhN2Wjgi+c9rP*S-2&r#Nz%e(2z$TX@G*oVG_e2*hs>hBiWS(Oc_bAgmy_hg?35E6+*I#=ukqp7#)sd5(?3w z+b>)k&_#KO2X8E_=TRY#2m6^5VuwH+QqTw^Wll3w{xvi2;z9^#WJ!lg+#pzHin;i- zz;iCegqlW;xO$~c8TKR*oRaDfi*nJ|SXg-j!X5$2I7w?RGYK$Lu@RUFOKi0BVVY*K z5rI;p#iSAD{sdC)_{JF@fN9=Z%)^xgk63Y$S_sM#7GqGBWI{f`s$tIx0E$*ph<4S( zk^){MjLtUGM(e(kq^Q&*cHvXNy&jgvAy#50V2+%TiR{W&N!cmAI_fx{cVT)IU|qMvJ{{{>jQgwQP#${<#x$ zxoi}bm(BmNqG}QR1v&VR{;L&TMer9;dCvaLipm9lp|x0UI?{2OC>i&;BS{>Ow98$U zjuk1(-CpiLQx|KVBE*}=DT4Dsm?En754PtEjK}TyLgtT$!hG>`*{#6b#t8~DzMm%L zenWOaqS_~_oA#dP8J(cZeWIfBp65C%s;GE18%DnRc;;*^PeWowP9yp+6N_S$D9C$}y1E@6X907C8wEA1B7 z7+MKWSjh*UUTmdtC$E#TIai1Kn4%v{b)7R_mAN08?j5C4?B7x9s^URpHo;{f!hMHi zXA27@S1N=iJWi%cXwqjgb(!Af{iwM!&@kAK2*NcRpF#y*J}^exGZD?})nSb;AWPMo9 z`#>=5!_hWM<{A+s`lWxO-k>K^gUR*}VfcI0{$rVv$~5K_ zy-Wkp!VGXvldbrp<5&k%#A-3q)m=k2UvMF9a2q{TQ*xXqSjoTX;Z|c8+?yV-N0*1w zktOHwPZ`l2yrulOs%oP=+LEH;Xp2SpldPIA(pElm*$cF(ktpLPjzo^QWF)$6rw7G& z{H#@DosLI-1lTtB>SN!kGY$f^SSZOj9{=|F6=xGxCg~}TeFI>)A9DtR>WI3|9#m?^5PwDJ#wpS0^QA_V z2dg)`is*K3ExKJ>i|(?mMRqI=lZqPucy(OuOEy7tbw&x&eeKhJmDnC5NS5mjqF z4}ShZ;oqNN6-_sutRyd9|NgDUE_C4EpJqg9^zYYog08(6_>Wdp8+(C0 zW>gm{+DpDKaenr*c>Bk`kM%sO0HYshgbf09zg7EK-)%)4{eU7audI`z;>tP}E#it_ zkGr3b-2=_v{OWlEJ#D>@6)tV4zn^Dy*{>YP_p=sQknLxYITc@6HVUKZ7TXr?SH9G$ zj;LpHy$Xp&MyvGvlUBr0&x*LL=cK5pXDxC(g<}BhG3^JVCm+onBmQ#5$esYe(+Z}u za-_tyy5`h&%DIfslCe46+(|4_ZTilgd^6AN=a{3kH*ZS~5{f%9w3X6MOkqFf{P>H_ z?SIpn)qvaG=7{!039PoYC$8ULYejr&#O+(7K9qVP#(Ry&88Y5C{JuMnOX^3HfnNE3 z9{=TDzIOyWOJjO+(dtn8tyWn<34ZOEpq2e$iu~6}#x?`XZlO2faSRrgBk`8!%aVZ|fw zXjLqD%OnC#clfSC89DLD!a-M3RGd>{QGN!d`J4Cm^R%F7pR~P=?6U)Z!XXxi2e_>C zCmq~xd9R#&ZoI+7w<@=ACA7K^Skb`h9Gc9dJP7l|LZYjVN~!Q!qP1ufEqqq}<{Jk4 zHK^p%$rj1u>rnmi z)}i{s)}i_zTZiiO)}i_nGb%q2cAVLX=epV&S>K`O;@P1#Rbb3Z&X{Ff#=u`$TYr+RrB@F$ z*>6GQUmgNnGKyu67{Vxa(8$8G=#iMguxc$-BfyKFp$(p`XB9|olCBUvW-6u3oT|#4 zL8fy>QtVxks|we~Cg>&MzC(Wu!sK2`fv|zxl8~9HB#*N_w8s<(ULyz)C2AeuX4!$7 z+%>BIbW<+;F{|&6oja!2U&Qss{Z`q&gME(3!D17b7@;zy427XoY-e`(2|xvjV0DO8 z|HVqx2dfbQ=&lGojaUeZP;@T+S1b0nyU&28JX4vgIRSz>fq%0Sc&qyaX_3HMPJm!e z;NR^8@T8VlZYHAjoB+X`z?ba=(6&q;&j?OXFZBCqOVK&}S!rRbQDvBPT#GC(v&vfS1vg2^`A_5X=c&Z6~n1 ziokqMfM8DG8an~Jsi)k7<2eC>Ie|yo3E%}|WdaL10fIS!N7)JBHCbf>i#Y*;Ie|yp z3E-_FWdhBd0KuHV*VqYsT@`_)oB+X`z+>zLo=`<#IVV6cC$QU2;OnahoX!al%n3Z! zPGF#lz)DVlU{2s`?F61!MPM~2KrkopI6HxFs3NeI6Cjuqc)Xp!|5HU^JtsggC-8N4 z0)tfqdSsVV!*_x?fhX7rJgJI6Z%%+hV|CQsJ?mYxO(!|p?ZoL74-RT=wip2Z#(nA1AIL4oq*i=?X^}U zOKkTpZ8zLeJ5qP_o@Q=Gm z5K7KpR#=dozci(4Ae@}PtY!te+JB~GTda@W>EQfjzwA?lCS20F3t@pZ=1igv_ba6? z&R+^OFj!c_pTAshryh64rb;43~Vju^uV1a}Z^-V|VS^Sz-AU>1XwS9Q-_c?HOFFE7X@~W9c36K`hxIS(u>R$t{+ZW(*(t?o ztMAa)w^s_RYzQ^vlUYb%glF(Ctym3%_Q^Vh?R93k?e%N9(ancxO+kWeuSeuypbkvI zFl6VR84mSR56Tw^nNfdWNw$evSN?{|%G|h@_B{L`o^+6(c`2SOKl2hkUJl=|55@d* z-?t+aJ|V}SIDueR?<@Zk$0&co_{ECd1aibGf#%uyjEQho@ER)ue^#KbigKAEi}5A{ zLGcpm_j#!%RA?K;#j}Epz)VhnU{2t*Rs!*?AS2Mo2@uQ)yv|M_oE2mQj^zXh<^*1E zClJmGG6M5C0fIS!IXeL?RB}_Ll8@&E2<8NS%T56CN14DvPJm!e;0<;H;jADN(PB=3 zU{2t_+X?hm^`My(Aea+4W+xEN3bGz7;%GDK}KLTCqOVK@Mb%Ka8{5JSj!0z%n7{3P9U5W zWCYf80t9mcZ?zK$X9XF7^iU78Il-L3yq!QeE650>rzIdjFemUfJArUkkP%3?p&5^q z2)x}+Aeqo zCvZHeumOQ^R*?0ep0o!6f;oZrCKWay5Y7rR0%J*g5FnTncwbUs0|Mc!AS19ZX%7Mf za{}*ADr`U?oE2mQ#*_9SKrkopfuzC)1j1QCMqq!^9s~&H1U{Hl*nmJdE64~;B<(?f zU{2t7lL{LU2xkQufytyj2oTH(d?=}~0fB+4aWIv%2LXaPfrX^P1_Yj1MPMdr4*~>p z0wqoC-Bjv zLi-+Mv!$F0%%DqxO?GNaTbmV|woOuto3_gU(@op(FecL457wvetDPDMB1z_*M((IR zLI@oDMHW5`{a+ zxcGo`;5f2$P9eZQ&Kvx_@toa2ZS6{%sI4>y>YPe`Ip30BNdZp?j12>%BALi9r$yzL z0RPA@e{VczzZSJO#jnn({c&&$)*mM}%Wvk(COp5Af?xVk%9m+DzI1mgUjqE2eEEB0Uv?6;?eZ=A zviZPK*_Xwl+?Q$o=6#v?_Knvc4KQ~>{nw|A((S~5Em?6sWc}CgTP1K2_^)LnukB+o zoB!I1{gcmamfubIFX5MN>Ky#nA9lp=gUo*|;9`^WU+;v7lKtN%3$pd{amn$!g}VEW zse_lZIrT{jc#ptX;2n~8%Hjn&3zE4;K zMd3r@2vPvEb*5l;OkixuAT2Pv4_V4A0sfIE{syz(jD-7cil1f7bo{QMwvFGyzUDag z(v;tE5&^%Gg5UiD*D(L@&0%H-_z4jw8`2iS$V zlpFPbYn28*Mwbg9(Pzc6;c+q_L#+jX#bx6|<-D-o;>6Cchc`a2>%qAc{q^vv>oDRH zf8Ec5%wMndvT8Wpgnsb`lSG65<*ax;;wu`HR=urSiGR@^36viOsKQVdjJCpAh*d{t!SPUZ7T9Rg{iQ8GAF6m? zV`6oBqM<7P9;>2K<1M2ZGoyjNVXE#0L$F^JGL+4H1_-q4B!5?n{GytZCcTwSoL1F; zP>(Io_lDaZ>O55Z{e$%;iFrPnfojMsR6Y0 zDigG^I6KffD`tmo&JS%&x2-vSe+PD;K~8xX>@%Y&TTbHNud4L#erlxcg5ZB{f0ys&46Zo#uTXRV-wda<-FQqfZS#H8)M930QV z{Q`OlpfpLC+j>j^Po?m5a}V?QiF^MBV3tldd()aMl{Ww*?K|`?=!y6o9zoe)1zZ%C zG>wjl<(Oc+T~c|;vi1D+;60oV*xY%0;93CSzTLoIE<5?C&%7taR7icqsv051W;xDj zeDWR|{-_m=jKGKn94`W=UVqj^k@RPLv^?==V=T!0*|1dI$e(q3|M#gzL)qY? z9M@XWv|mlyw@OD&d`)4+Mv2KF7&*}<4Cf|dd2In`yH7#|}7ht`75$Yw}sodT#9q@B~JUR%s z2p%kZu-klM!hVQ8o4HXuvv&s8I*O-Mcjs?59Oluue7?~1Whq?j??(&%amg^o*1L`q z3TLiMewetzyK6s-Cl1~3zFTqjiUuy8)cN zimwIZFaB|X!ryV*M}7AXvN#;owt0Hs;1R7AxKJEFY{hYV&a3+(mOGYGWG}QL!`+m? zNgbFVg?f+z#7NmsFYSOh_$yXzup*_Y!e=YIOR=;O_G#7}U||f9h5;4>q^~Y}S+lCA zjlJA&#bL3RFIkaYEcT-D8?DgYADPM5OC0+7dWqwkHe3^{ZQx*oxBBsJ$#7xB9Yeri z+18nMa5GsR_LZ@;m&U+!!EIFVz7M~Z zm8@@O7zAyr!t7lP=igg>tXV)2@rlXA*B$qZLAGDAELA(RCt_JeFQuClKb4wfc5F|nSm1d+kNb@DK!41XCvGx6U|3CI zHX0IP#|R3S08mXkywwq=qSk;VV=$Fz$hSE_*CLOGCu z2uh$$*`OFov_YC!(FS$?8OA#%oc-}-F^T=vSdiJ@kW^J?&kDk5@My5CBrjMN=c)HV zUt%4!Jo=(FUXBd&4grS!&<%vILw=Ej-2AF zUR(7*tc)BRtgOaBI9gd{w3ysy5^Iq8y$1ZLe}%qs|0>201L}vc#YadDTn|R-hAW+b zS96c|Lxk01)k4h%jASR!cq8FYF%tf$FcKepYICuWB!nuKL$bsKqY4Oj?wwuQJ9}rD z6ETi9s928sv53~3O%zG79QR>0nyV@=m_o6N=XSbz$!ycJJbh>H>^-emZtrD8OHu#S zuX>rV`8o4O!nQs8t0(+JV zi|0K5UL^XUs@?dw7t%>)=5ZEe`&DizZQ;z!4dpG+2e;=a^5^}jfdPfR#d4Dop*kIyGiA@>4f%n3Kkv6+zk9!h=>C%^Y3 zziY|wIxjMZTTJfvkmF|d2EG9uFG^JYkTaC$#BJeNt0Hf7cJJVxdI=%3Ys}8EGP};~ z;f&Zx+#(5#y5F8rEL6sYWz3EK`;CDIe-kB$3U4+MJ>M`m&PjC&fNSWM0}AW3|7ELL zq@$yZ<||e-7#(w-$Amp)=K(98MOk(7C{VA_s)FQMeQr$MCc9IqhRab%(yI7erWQ$n zqMc6diE)}pdLp=6+){}h)LiYL`6KR&VXlB0%V^A>YR}|d#$)~>yOTGx(@Ne_w3YH! zX47Gz@*daqV=8aWRe75~R-DSajK=(-;!NIUJm#+ypMQBPvz7KerpsXLZ2lPl>}=wE z4?pND~iNp|Pj~(&@l2%oSE+ zztL)??WfjXwW7&fPnn%pS@Aqrr&jZ)S~KSo=51A*`Gc&PymhuxJm&ANZYb}qcT%C# z=QOpZAAV|k(2&YoXBm}~`BQb7yvumZU!*(#@>XWMIJJJtJiK)RPUdaZ#$8r4ZfZsC z^A`|q`PAy?(=j}K=vIVoeOf!U6Sq*(T`E+_*Qa&nghdw@bMg0>EWAQ!gQWwDxl2RI z;&f6{z_k>fZuYZI9eM7FW-0YgG>4ijSmDg&wY`ckpCjN{KV_im|oI1eLw0j+>`8-B9ZkznS~A+(!2H( z4f6XXf6k})gezc?y;xx1;nxV#a=YEWU71+&Jo11iDwXjV2$t7}1|<`~kmtsI#7u(E zk;TVg#iHY(O+ear_!>T-eAYjpJOwT_SnWIfk6-3<%PPoY>}v$)1x&ACauqL?Adndx zj*6g|0&b+RH;ZG)b7mo>)9yj{P?M!J3&PGUT#2BY7j$!jl0r#(HPM-cf(gGovrWJz zL72s9r2KPcLCBkh{DfIBP0T{FQ;O`}kk^nD!Ymev2KmEYy)MO9WdT>2MX}u;->yt7 zF^iffDwTs-NTxRnw$9ArYd2>WDow6o`VODT&0>{(jb^bX;1RitC|!lCz`G^AKu+_* z5-e6y2xujRy;-ax&zXgkdb0>MSxU1Y?99TIh`iSX9q#Tjg_3wbMwoDBp?a7b7(yD&f{-^0`3bXNnwW)TrxdCFy4R2t!Yq1;2Ki%Ny)MOLLV&BxqS$VqZ&xOk zn8mm!DwTs-NTxRnw$9Ar37azum1d<`aHBqvt2_Tl_I$)Fq!I)E2b3rPVFr?5AR zJ;-xrA*J3dLQR&^EC@TZa3vz|nxLB-loU$RsMeWkSuS@Y3XTVivQEWHq+m(qWW-%3rG7V-Sncghe zIx~wWZq6)Jnw4fT!@fqjHUz{wHjqYdA(bGINyIBO3rPV_rLZ@P8RR*$kWz0Jp(aad z7KEKyxDt_fL(t((DNLaxWx|<-f(gIJrK(Pe3j|>nOGu+x5b|arKVcS36SI))lp@tX z<259OFpJ|vgZze9uS@Z^WWZHsQEYe2w<{A%%wpaXmCC^^B-5J(TW4nRq|KRyO0&`| zPP4DkELH@>ZBty7jb-Yh~*meMQ;JF{>lBJUMJ zHz!|Aq**8v&MXv6_+4j{$bQcw0KzPGBaLQ3$eV@ygjq06%tEqLid6q`uOTUfSqLrU zFL?F36kjL?TxAx;c8k7UnOI^LO;1!R2eXh&Zx(EwnZ?lN%tEDEX%<5iS(NLrfVe%5 zG@6A}f3MDBM&MXv6 z_^q=^WPgkx%wj*%XcmOLS;$YA1=GYVBs--@vsm&Pl0uk8ooJB1?A7biBT}!*EQ;+; z`*vkwiCL_8qEb1Sg=BiOVC&2*p1L`+P-#}0#U%S0V1AStywBdyGvY`||{07K=!u zSrGDOAwOXjOcS$^?35zazv?w4g)oa_|ChacfwyYv|GvMlQ_fOSAt9+0DTgB26qQOz zQXx5)q;w!8J2_O6D3n7&h>|2!NRm{D97`mJ4is`O$LBM?-#N!#yY|hx{4e)&Klkgo zUa!k~&oO?#-yCzyvF4h4wf5Sf4DIWa)9c4!iYsneY?`+r$=iHqS<51G06Srd6`U>$ zH73gP?nK8X0=SvIyH=7Vh!WQ04fQ!|=#Sii7a7h?`mCU8B%=Z?QgXvMfsJH+vXJ znK0b2l`IR9Wf2~;mW7(L_ZUupspJf)lVzcO(Y~|VdEY{hyl=r{am!-Uyz)uj<~z$; z7L^sm1vxyX%R-IGvIyBlmPOrvwk+ao7P~CUhbb#u7L~QhKVtD%c(qVan5f+d4)Yuy z!vtk-C(EKdZC)1YO_oI(8G6IA2-{v3?oqkQD#vf?g-1?O9E6ue+{_wpJ%z?)ksmf$ z7VUW~yjp~GdP;Td-Z+ve-1QUXr)@ z&a#$8ezH>qr^`Z($+8I9MV7^3|7=;r*(`QhbP2~=xGZ{Vlix_^v2a-^hzV)0kJ=}X zYUEQ`UR4DGFx)9Xjxw_v-tWwB{q`y_Ak zon!O`a9(E@-JGvYeleQI!d0Ri=4Zn0 zw9>;q0OsAy~!#`BSUXkC1KmE#66n+*(x`xiXJ&sWO?cd@@U3w*XAdA8SeNAdsgp&sB$T1OXL9oW z$h#VB7q@6O&Fh!sZN9UtMN^RMRKe+@QDd@bLUxfw)8L;inmD;$G_MwY546-CMW%V@ zqMPNl$*+s^Sh#4E!~AU6lUx>!9@)-R%GypAO)1*EXw;i5nlv)>hD8&$y=dH{>EBu9 zc!dv-oU=FxFPgZSwMwcfG``~0F1lGSY_e#Y@>sZN!d9|qL>5hW%vv;R&K^Km9D|ed zr%o14wNQrk5y{E-Bky^zUEHGCG;eg0xB1Sp7R|V1rwUFNjT)0h6S9jen*4vZXyW7+ zyJ*^nV=a7cb}qWvMVtKH10D+(jdGZurj^NM(dd!w*4f+1qG?Z?7ma$8MUzH`-mqxG zwik_iH2s}bj^8N^kDRkO2rrtrnKk@=3XO}Vpy=k{u*sqs&13kVZ)U?*vS>sWO?b>& zG-}QsKv*0T((@P6Sv38^Bkfa?lkZ2~5n;QyMWdowdDD};&3BfyXl5rnRdBj!)R-)q zkX>ZaH2PPWddzpCF)I9 zNg5e?!zu~eUM24Fe5>53DpSHECn*lXt0Zn_jd#96<0^SGY_crY@EHE*Wf8WLWg)UG z!eiE5i<-0d7*7BEp(uq5aL|^!kx^MA$BFS!|lOB+1)+XIaZ)MY2-`r^`Z( z$+8I9MV3XAf3_^*Y!-XhvLPI6;j+l=8#eiE3?2)Yg@TxnHC5>4;-N>j*Jp1h%VGm< zUKZ+2mPHyFdc(2^+g=v#(R^q23*|>u(PQ=?ye#6Fvc_9Jq!bRna@b^9)Z?*mS%j@* zS%@r)@R+qM)SSJ?aQfFIXGonai}ImNIO^o|`jK};*e-5aY?`+r$=iHqS<51$k5=%a z9)iNOT*!Hq;kLJ6z%JE5#@W@GugYdG5n_1)SqRCo4DCCso%hRB&RGfM* zsmH=kK%0N%T+=X|!q0rSW^=R8d?%kz8^;@*nuez7?@xtK6l8rqZAy3tw8(E!j?R9I zG}qssy0I~zt7ZoH)uhIJb_%r);%{v4z(v&(AmWJ$)Mik$&f~d-tc4y+dEx5Ym2b&w2s!&{19+c zI2h``88+zGBil=|w+H$Ch847Vf9g&4mqv!(u)naq)3wc#{kd2B3nwTn^zbS1;PA)r z_#tx3bgKowh7{Vv?|xN|A34j4FAU%gO~rWM)iq<4A|>NzHm}NYo%WqwBtJ|{_}z*k zqYjPzXiWaxvHBT#Xwp%|oes!#r#BYasb(e4(HA%Ifx0isr=O~$WIAOkq^U(}hwqlo+HsM-usjQrH5-Q(8V4thaTK9(1dZYQQI!!ssk&JZ zOFINLJ@UoyrKV$Z9Pm!pUBIV7!n7oKD;acrf}n8GLCAX@pIFa9Nlk8l(|@#0qY6E8NVWwO)sdt>?-2tRhE zI}VLuBkW*A{Lq5uHg3m-^-QjZn*GGvyiEA@&^SUylpL3KQ{=bN^u&w#n>_KJ%e9K0 zc(IBWN}MQd70$-yw+nhfFT7m{w*;HrE+pSaGJV8e$&tkSg%3*>xnCGZK+*e!g3yw_ zUh4%a{mMp0#*c=2ju*KWhhK!s52vehe8KbSirevYy^-q-WuLCi%eAUurE?q)i`TMSNe*8W_i%WO^Vynn zl{~-F!-!4sFh0roFyaE|nVVLQWj@WdOyQJnUgWb}i#%IorosifHq%!0!p2;NMomES zmoiu7T7TT%oic953V&NAo|n!4NPFz^lQf;SKDV~hKJ5TX&)2CKEPqhm_ z*pi;7*c8vxHd*(ZfBEZd+2?8VGTY`_rtmy%USzvmi)?b9eCt#yo)w;A`D}zAee}q7 z(_~wZY`12cr(pVLbJ81bt`=#PK^wO0vjac+x2GXo@78XYY~YOkoCzI+`Bya_!bLak zKJ<#?94VKb6D!2v6oB7nHwSA90#DdmdM=u-PRyHpTU^XO83V zo4r0ZFHoZ|4NPLa*pYj+Is z)%^aVy2IN&^--UE_uQy5e4$P9?oWQxLH(=bXZn;&{?dJ%p73tb+HIP?!b3RQYahxU zN&4oS2dD4(i%o^!XftST>vaCI|7z2qbv^x!Vch9vzt>g3<}uS(@$64tE@xDuB_3-! z($e9W^J8;s53)6(toHE679%e_K4wVsz>kOuNdLZ5Xbf+D@dP+#%IQ-N3PszyX!x-* z#KuYIbm)72(jyB`Do5zZDbpp+Q;VCz8&hd~m!#rr$q{*T(A=Ksyrf?Mk$k`1pmoXl zFHv+(|4VD&fMhZ@T?3kggQlKdIJ0Y?X zPoD{eIf8^P*MKvlG8RnnU*~3cw6lkH%X2Um5Gc|4P+E|MRMaw?F>Aj7`pey<96!j_3bxD%Q`v zVv#jo%=-PCJsp$#p3-N;-&8#Izh1G?f4ySk|GZ)j*MTOvR{f8jTTOGXT4XJRYg#eu zA&28QDfd05Ya)lLZU6JCf3wos=UTDwc~^9t#Xg%m=34d9|K;mTxb^uDX6BsS_xK-F zz2IM|+Vx+l+WlXt+T(v(wb=F1=U=IM*}qctihreQ!M{@V+JB|$;D4p+h<~N(=zpc^ zz5hzp@&8KINB)(nQ**Dn$t&pJzJ3;ag?Kji{S|*bFShFJf2Hc&|7q1?e%RV1_cX8Mv-MOCbFHtMVx8lZt-j zOy5n@SEm-fW5U1phJPsHW0<*}&lkh{^y&L@Mc+IfIJbZ*o4&=9tmlF2GWy!9Fuw3t zce2K}bLc4YNy-f+!f4X{@J|E&6vR!#s)Rix@AL7`CLxS>eDxUq*(BN4BisGhj==@0 z84HbL46$!;lBh=#MAJQl? z<>7CM9>Sr-BP;CHTUFHaz3~XE&P2oQuhD{A7zE76(J1=*I;m7Y%%bxPe z{QGj8sR=rKJ~bU48NifeTaRo{Pe%|sXNS(hbC;i&RcL;as7GSoOt$sN_7b*}DGxQn zl&c(59=ZyrJXY0|$H8tgXa3~Zk^hA$UlB%AIOSop=#)3ebtiWfJI14OP5+B!v8L!4 z!)DPjHq3P=MVCdf1J3!)r@t8R`l164n?(nl^ZQYoPinZlHO}>T|Kjq-%G(fzS@_S8 z;fl8DI+>UF%$TQgmVeUwXakkn3#x#Z&WduXj1W zc~^AGH#y0^KzI5t4ES#^erM*o69$|h?b_*# zhQ(4ZbQUh8&e^wj`AMQ4iEWx}>yho&Y`0+V;Yh-5btoL}@l=!56#CQS<>St2X|1Nh z5pT8@d=*u7ad@s3C&!ALo%pOn*H^En&!NI6^yWVVa_F00_`KM|2intBn)O5G z<`w>Wv3DuDc*17U#WOP3{fyr1su`B~?YXA^#jQxsqGJr3MaOtYt~=4KNZ~>X%PO2D zRV#M*cjsF9FNWW*=rU<}Zsq%BWn8W`{`%GFzFf=j>XhD|6mySM5DvNU z9%%%doBqI*ydStTJJf81&;Ic-$^P8@^w9?jlfTDDb8Ua1ur{H*Sg^h6=Pn1$&EK?q z`t#WTL7xvR`YvDc{@CBYzvnM0&*|^m`|HiqdskjGli$8gU(C`!V3_mYx38J&a>)K| zqRs!_an7Hw;&a&C1EjwL5^gqXv%#I`U;jPVI=K#WTzJOE@i9T^Totp1HJOELIGl#; zMHb)R{D1TO*+oDYCpf8`THYk&v=s;zpQ`tC?KHdzj|~DEnJE6m8YwAP&b@e@S8Vt*$BV%82&6bzx2pgc`B59>GY;2`Gb0K zJO0F6H){lcb9>2;ZyFJo?r?BE`t;s5YhSVU;knjkIrG&#n~piw$@#;e(?gwN#+Ws@ zFt1%oWis>lEb`XJgg-#eZ$5jh}^HVqb55sL<=J;F(uXiP4ewelP{0g~djhE@dDGRNeO<8Ev zl=+^ud9FKsDOB1dxkc{D+0*&ISPAK~CmrAjo;5GA;b%=a7K)76DSd8*pB1vUClQkwbIr&dl%%o*CJ1?&B}DiwTz!yX}+FXTuwKi z@Ai5H+2m>z8nfoRd#*e6Q>$q5#knVkr&c;*-A?Cg5XIb1>s7Asb~?OHX5UW#_dg#J zezLrtKNOIc;k#@z5Add4UMBpS!zi_d*Pvcf_=ndsb_u=VUo;+7+ZfcLAhbH4LyM%JQ6FXG@#@lBDvLf-#$KpEs*b{U zLgrVfNdn=7Re|sR$%iic79b5P#-0!J!ZS;XMPC?;I zO7k*jq8Fsz=zYaqftE;DqXDWi4BaHXjGVC)eWbV*=woRma>jSa8T;`@x6kFVXqEPM z0h%NALY}o7V(%ohK)tJDFN4jj(>_X~Z=~&!_f`RUZ*`I9`8?$PUW2^fhtWb+{up_` z8_*xB_b1wdi&tJIk2fP-xioU+1CT5CMAKE}dNfnI6O~u)eW;@J5OS3#k*mCfW+~%c zG*NRJ{<+9!>v!~=dUxWkbuU#ZhxV3sN3K^5xn4aqLm9`SUdM*fH$h%?;f)`U^daPt z&P26U??qHcnulEZZREqjM%5H~ zR3lOH>_)4!k9X0T+Q(<;3$^`-j#S%rr81fOrM=O3sU>Qx5wu0eNga^Kaz64{2BYKE zdoyYxjf!#i#kgtc0ac!Z8c46B&eHp6sE*@9yUFNAm_f0oVyQS zc+^I5bx~WX4LU_S6*>1pbf4zA2l6~$gFMf3QS$@BSZeLSvqCxsU7#w*BUd>s_Vz&C z)Y~Wa4n{rHb~Cy}8il-a?n7QVAEV>7k3Z3Mirbni#r4v5$Qk94Gs0h;YNCv$Xo%uk zp&O;P$QfrLXWWT8DdT?BO%DbH z-rLpa6`jjl(In|!bU**k%RIt==c(-_^q|@n^52K0H~6pXeTZCd9rBfHTVC#cHEV?K z)vI$$bgk;ON3Tm4AiqXkgdSI0Z{%07f#^E5-HPUE9o&Ju4rU`iXBw7HpKoWPs+x^U z(EidOR83lowv>KHdD02oYy6~rbVN6*?Naos+U`N$YQOg*@AoO>^|uoF%2kf9S@CPi z$*7^KoQ?dNav6G0vo-)Nkp>~p-w5RSn}9rm>B#%os%$2cuX@eU(NcS~m(&}zm99pa zM&Z0&i;|Hdk8c(7`1a-g(BnHa+$k!zCGy@zB4^A+&e)<{nz1u-#y%+gC8xa1p{S&) zoR5x@`k-IbI{@vdQ4K;K)kx&MEkx%j<1;i(`UU;087jG3Ci9fqc1Ii4wh#JM+8_DO z=n&*Pqtnn8+V2IZ1nVO&b1CYtwgU8(>J3F#sqGe2QgPugemL$?^tIxq#kgnDHi}yq zM|2U|S8=`3eo{Z=QC){Ts?o@=QIDZ&ikpdMX|7&F zHP!YuI#61ITxBJ4l?}+pwiQ25`PgQXCaZMunPtRH%hVg_2RBWK<{_ z6-q{hdU;eR85K%Kg_2RBWK`&6j|wHD+AEFALf3dy zC>a&QWZ794#=VivnxXCDdFMD2QdW?GkjaA&D7`FtK zR@|pC?hACU;x@#%E%)Y{sJQKs$5#f0zsQ)E*$4SJYN9fVJ1oW>g~EM9UZzEiYlq4z zu5*mL9F0=k-Kb@~knsZYig*Y8p}Aj<{*>0Br8;k4A)mM3k zKC1G*X+{U+{a%c`5(l8~G`{PRSK??iRAZTdyb_;7KPv7e^s_V{c_qG!yb?b}8`b*_ z^6SS|Rd{7r+qUR;X(!~|J&aMnR=wj)6)KkjrpVljh zE>YW#sJFB$>MQLNZKd8XQ9bp3i|R{1A)mLcs;B3zA#z3=)KGC9&@oacsojZcqh&CWyte<1M)mCL4MABhe~K~Wf=Lbs=OyEskVdBZECBB?vTPy zCf@G}$ooA5`JLYIleRSkZKEpppu3bC-eDRmJ%U{2Y2+&NP$^}+kM2|4ax`8FRhxj5qw$m#5c+85~t!W^dX&mRr%I>b(~|t=<`EvfAdMDQa60q_(eO+m^L5 znW^e6hn`mNo@j=&5AvuELLSwzXbofJot>yoQC-p2RL{%wMwN7q`XQgA!N})m5=!nF z&~D0DjgotY1LHjdD#K5u+%ur$o&oKnj3ZET&w$D*u62y-i1t<7xiPLA+D&m+#JFow z6~zsYakrt}6*oS{O-B1EZhDM+0aa4mq8PUXRaM-lG42agS#cX;+?EIN-h<+{N69?{ z+DCEwpyZwbRa4wyG43c-MR6@+Tsu@Kn5eZ^HoKHe(m5XBuB;|@hfDz0&iI}shKxb`vb zOmvjuE{<{i&|!+ZCdLgx`HCAIl@s>nQ6ju)Ucq^kL6jvk09gLbQE731ckhKhSD#w|rBD{f7U`wktgxZh&jR&{xIT5-D| zAMYM0{23jtVaU((TIf{8HHdM?qhl1;I>w!f!uxpKm&CZ9=vc*F5#t7;(-b!%#@&e; zDQn~2U(+{_p^7d2Mgq8RrsI#Y3<$GET2af;gzh5g6BM@)Ic_oPRyM?~K#uzoHCNmZ$Z@}-i^_($Z4XW3%AyvEtArf4AG%m^ zhs3yssHNhX#<*6fr{X%rxGv~K#r2ADm!n>a8yw?qMXeNfcZ_=g^-23F|IEfsJLrm+;DV?;zq}~dr^2NoA(4`+;r4VadTqa0(8CNmc+PE z(W#1C7vsK1;g6u?Wwty#J>Kn6d&QMQKHfdiP{q}ZafhJ}ip!62P0%pKwTp3`QAfpH z7~?KMH!H3n#tlWMDQ;AZy9t&jC&c4RNOl;ZaF$bacg4SHoS|l zi{k2|vC?tqO5NdgK;aK*9gz3i9lfQtKIm=fa^&3Wk#p}w@2Ynq za_$RgiQ49&rP8a&xl539zeUT``#W-O=_4|kq*U?6`Ek&;P75Y_eKcL^GjmY)3J~FMh7usUiaDER! z&OI9CsqJ`FQfiKz+a7r=m!KWhdlho-Eof)8jYhjj_aNs!ik$lr+D*N0Am@IDDyr>E zw7c{Ta_(=)xw|&tPFlTtBj+B9s;I32+D|$LIrl{5+zU{3_4Y>2y#duy+i-M{G!i*? z9CGfPDEwPkUgiVjx!?LIu1IRz0kxAVAXl!1I;ias)KRLBT={t9%BQ2#)q4)=B=tb9 zG6;26+fa12bTe|5yOFC*Mdzw_20CAQ8M(?*)KzUO(1p@Uhp zjBZu$1!$Di8@b92=nl0FM|Vmik*kbDuJSaxN4?LZvC;zMDj%Wy)b=^LU-|;M%1_8u zw$JBUtllzcytEf`mAdFbwH<*Tl8#2M(gL|k0eV=yLy_0>(`c&No<~ng3s5pXG)--v zqo<@VP%=K0jPDpeKcwC=Xr{CmO2&ttQ`-^f1?gy%j1RfW*=UY>yP}t+zQ|REp!sSW zfnJqvL#}c^a+T@mb@jf0-jrTNuJSQ@TWzb*JJMR@DjSfiR6dsLy?SdP&($Hw?}r_a zK2mRMG)t@aY_vjcUD0RK#mJSfM6NsqeW~7=$dy;1Z`Jl?Y%AA@_nXRvXUd*vd#O6| zaU6wqR9j7t#+E$>Wq?O3I-y!F2$GcPc>fH@F_W;yLZFNv%>2T!S#>ly6peE|=iky2DYNobp zQFCbsa_;TOxsRik>U|bD_YKrqZSSEr(lX@SHORStptkBQaXha$QfcHW`=JhMtBFpN z4o0rh5V^{wsH=LfLKjLykgME-x~pwG>LERXT;*xxDzBkS)cZE-Eq#n!<$H9g+I~U( zq(70XY~LiUvJbjKz12~F=}_b<&Cu0qYmEj|g7hbHl`>6vPgQLd(Nt+K}uP z+IpfnQeWgM*C1CJjpnKMUbH}(h+O3bv`B69&|>LTMKKS(u^XCoi|sJ15PC#eN;l@7>NdZ3@xdnwu|U58v{ z4EjZF zMBeX#sD#=MMd9;I{A)P!ew!fgw=F8A-cHE-y&P?;wgJfd9fG{yF=%`Bjzc?0k0bB* zCA6d37NVV`H<0)HA@Y7JwBS`kz55~O)<@;kmXFFy$06soLC)=lDya7|aC1sX_eGL zE!0*YwUQblkE%KHs0vU=^$tZI)hLua%g`C>oq}9>4mwk93(#58BIL@;kSni6uJSuN zM{y-vrBy1R^VC)uh0o6AW%fg^QU|%pvB*{0qOOYTgglm>=pwaUhPq4rk*f?rt}+I> z%0$#taZe#vc?I=S+alCQdKAy>HrJ+8K~Xo~aza+N8_Rpy~5)cZP`DlJ8>@)eq< zwja<_(njPeTenH8?184McV9F^ItaN+BlMiwPC&DyR>)OOM;=RGG+Vs`k#lcDFRJZs zG)KA*Id?L0?knge^}daqyBf_^+t=u2>3ihd%*knPE%b_d4@bVTwm}Qjb}I6f^=#xT zYghEDdV3>h3`ei4Z6tEWSmcZe=uP!bMP3IVB7Y*_Pt>6xtb;w<^2(&Rn#jkIkA78K z6ZD(Z0{J*PAkRTB^rw0UqGbNi7Ue_byU|wCeJGhf)4h zXX#etG2e?k=4a6P%9xA%oL-7{QN}7XOIOwnXjirUiONZN?b3Rsk?U1K{_NYq$o1+Y z|3=0s$Rq87JktKCg6a)IyGw(SM|vA_<;RdmIup6_95hzr`vCb<)N9b5s`4Z9=u4i; zJ6>wr5$!FNLEcAYYa_MXdkbjs?ux7 zds~XUw{^(Bt+NsNh_+~-&efjCb5$F8t{S6i<-;0kj%rA)kt?5pT)8juTwQ})c_9wvQ^7Y_EbguT+4*AN^1$ox`A$OAu)OJp6>x=rUtv?zd4Ghvb8jf7=J~U9hlacGqM%SqA_1N|?x>jwg(Dl+< zuq-iXF(=M!WseHYlYT*d722^Audr&{6)lh|B9CBy9@APY)j9ortMX{QdTSt$pdtEBZO5S>r4x`xa4PZ$x}y#1?Sp=i1|e6u1BKre z$;*sInF>JE ztC6exib|_(%d@#JkxC+0DT`dC1}dxGgVC;1K5~_|sG{0VLwiV_k*jn^t}+N!Qtwa{ zeorSaGX}ZJW2lPSrlI|$8OT-UAy@ej)llzesHXHCa+R&m;nP`a+W{Rc?SfooFXSqR zqC?c%039YZN3L=vs;{>5Q3I(Pa+Q9_RfeO6>b(u+OXHENOh?D4?FDqKG#9zbV&p2T zQ6u$!jT%e8BUdSXE}v^rTX}T6v?p?v8pu`hQ4{qxK~1H0$W<;xC#dZb)Lgn0xym5q zDr3-z>K%t#OOGR0c?q4QwuR_q=?&y6A0k)z4xOUjji{Ye>b$hd9;k!b_C+0~YRFX% zMXu5uouS?~sFQRS@>u$!&T8wA&XxuuR~e36Wjs1py%W)S(oEzki%=J}y^AiCK0vOr z8oA2v=pyyzozE+=R0g@q{-~GQ4n*NM#_}?Ck*nk*S80bXQ|}q*O6j5?-Io-g0cyJe zh2J*g^FyIc*U2%+RUSvzs`qJhz4S72m8EEi+E$>U(n{ni-yv7o@`CgpX?yhNaba(z zkzYgiMt%)#jc!zxj>tb57NDEeHZ-=4LcXVY2;HLIDQK2PKL?Fe+X8f(vFc0z4^$M+n}*(>wv~doscVcL#{jyJ*eJCktkc1p~uzR8M$&V^n}_jN7JMM$dzwI zt~?vPpx%YZmES?Gyb8@#?|S6QTVBYWjM}zA3#1*9EANh6xixx2y&aJ&pNm}iQnXmT z1;~|eMenL@40=x*i(Gjka^+QMje5UAUq~B~SLY7hxKmJDS@e}u0l7+5J3 zgxoo(jM^5UveF{ty)8rD+s@s&%~5Yf zAlg&CgHiZx*1XIp0#rl2Ly;>_LABI2Gq$Zjb=3A{Z2JLuhPLj({egOSMjpXFDEy{w zUZy&#Cmo19f+LYf&=WOO?-j@+xE8tc?I>TpV zEpp|u$d#+2X6mhtT=`gZg4&v)=F*ACl}|&iJOZ7l-aC;ik4LWjG-{>Z*~pdOLT%Kx z6rC(BN3OgUxpL)8_#C5pYamx{joPcNV{9ux9n>~7wk-&41!2aQBmXS^H9B2!KOz4t zUaD96vv_%Q)&U`-W@uAf-Owi0LmuC8$m45|e3aeMxr)02`6!2>3)D6eb&*CRS9uV* z%5>zPO~eu|R!8G5J3)(|D{GobF;M>~|f&w!Hm8IX_dCe%Z5 zqtGSN801ksggmMl$j7z_^;X<6)A6pgVV>=KXuVbr= zd~A)7k8KF@c^iv-Y?INI+Q-w#$2K4N*p?t4+gE6SM}-PJs?etE-`0KOsF08CKr~2k zM3+M*#+2jnW}BhS#~$j5dq^05s^?R0D-k&o>Gml=Yd zm2N|>@(6lfZBx(-(o@J)<{(#Dg663AV>DMkGl^2k*VL<`h= zC|W2TgIwhl^t#$kM{h`HAy>H=xyrR@v3hSrZ%KC`SDA?3Rohhbp7acImAS}OK0r&< z`w3bmeT7_Qi+;SXqP9|KxwHdvm5Rt!YNLTD46>>!caTRpudA`4D}h-p|l_=^NxKTV2k(Bx>6h{UGgxTxAdB zDs|A0>aCA{l8!^J(gFRfw$5mSbRKe*UdUAjqmAml8T}&NgQ@txuX7{j)zeTQ6;tKvXNNwArJgGEtl}gA}4nZZ=dnDRMYJyzlG!(wOg5TFb zJ4zQISLuUX|W^R8GBXk#qk<71Xx%mE37b+ac$c zN6tMM?XKRVkaJH$mDJWA?IoRooO>a1?jTfIy*DH0K7jUB+oPz8^f+?vbI7@Gq5agm z6zwmqM6R+CRaaZ4Klf5n3FIofAXlk|4p8qwsHSuza+Ox7mfG5(1EtfDt8_uGG5{T< z-s@0p=@#TF528A1n}q60Pas#Bg!6m zqo(R@iJD0rkgIe@C#bCtYA#)lT;+P?DtDt6>U{vUlpaT}@)9~xZ3|Ir=?&y6A0k)z z4z*G5M$}d+F(9o{0iCM0%BZ8XA99sC$W@x6PU>xi&Xi6=9!n2&p4u)&7f4qkR~d|4 zWi0Bd-Urcz(i6y4UPj&3_8RITEk>@g9J$I5sHb{=MZKhwSEp5WM}5?`59%xJk6h&t zW91L~p3= zSoEgU1i4CE22gHtI_9b z`x>p3zDKT-xhAbr4y{)2p6CmyI&zhx&|0-MM(d6h|tBge7 ztL;wogLE%)l}X4|=AfU|y8vyF-bJqR1=^^#_2?JrN8~D74oa)kM_cU?R_AfZxgAgm zwRJ`%rSp(;dm)eDMzoE3??BF-h_+MPR205nDKGO3a_(H@+>g+X>iq&a_jk0j+VZaD z-3nyfM6jcTa(0aR0Z9J$I%sFvCm zq64KjkgI%%T;)4-uzEM5I#P-2(kd0uA!@6P4wLpnu2KiNN>g;WdRw9T(rL(J>4A<^ z+oh<1bR}|?!N^s{qNCOOAUZ~R0=de|sFB)UL*aX@@-mB&t1L&Z@&jt7-e1uPQpxMn zD!Zc=YTE~$DD972xbG)S0Ps!idMXsDTxA7vm7maA>ir#^BW-g-T4hgkp4zIQ^QG#@RSrY0(h7A|?`bGG zGpL)|E=3nfS0d*QM$R3JdZ_n7)Khu_C1(a*qPEviFKIDymF37)en7p|`zz`rl^mQ_ z*&X#&+dk+rX@BG@hagvJiTbIx1M&#Eqs!IS2VEguj+}cva_-aUYV|&k3ZwY^Leb_5zC9gSS21#*?M(J=LP4Q*0i ze`8}tP!*fT_LSTFR98;$iKmzxskQ2 zjBSyBbGI~_tGLR@aW&8diaR969f@96+zBzRE$X7UPBHF0G*5ATV%z}KRdIu3+|B3} z#f^<|6VQcjQb2NP~3Mh?l*Li; zDLt#@k>3^C7x`VGT4==aVZU{d-xWF*CGU8lWK<{_75d1dLdmF5GAfje3VrHPp_LvL zN=7w29n~&qwR-nL{*=Z3=u5R765AS}uhiBoww;8&QCp|j)(7=IARN(&yuxi!(x%BYJrNcE6&k3-JwjQT30 zZ`9DJd!uHe0g8J!YJJppBhucgQ4P`6%4mVsNFAfPL|qznE%G|J8F?LyLE$%#Sbxat zU^4PL*oeFicDyBB2bIw8I^L?t>!2R;IyfGAuG%55hzrr5%D4n=v1i!ZrO0zN2zjnX zpsmz<5Auq568UVsg7Vb62$hiDMy|2~xylx|rgODN)FDwVqRx%#k35q@k!Nx=@?1TD zJXepQ7d2N?k>_d-@?32*GM%eRXqB#$RZ&Tev@Y^ooq{}9=OE8jZ{)eU0d1>{;b?nl zB=S*?Lq5vK&<^UIi9Aebgt@0ofOqIYGBkDxTx+BlkRmd}R3))R_ zqfvS39^_+t6#3YmK^4_I7kP$0K)yzNgFIJT-Oj%NDsEd;N!kgy-X6&H4njS26*wCC zDsVE|TNxeEzS5b4^7a`C1BdCL3XJ#SK_##wIV}1{L#@8Uv_)o~kyY=XF?kl1Ll(9FeB~?W} z-nz)gn~x4sZ%gF4KL>fnFGB~bcL1s@4MMIm0=dc)p-pqOAnKE-jZvk?q;b`e=jw3e zxjGJcu396{RY&x@=ITu3x$1#DS2Iut&DA30XYzaK2#xeJ-%KFTxDQR=-Ad9DhO=V}znSMOcu7-<}GmB)~)EDCL!t5s2dM3uWM z?L9E+c;vZii#%6nBhOWLiDqi_`RGi=H9?)F7Rblj0r_~lqGy#c06F(obdEB{pmU|M$hi}d zbDu@dornCn`nQpv6`!H=l)D!BOqaSRJ=5ip&u>lCMR9deSE(NI-i|}wTN`wdde21u z#?Rj9Vzpg?dPr9z=MF>89glpanT~qx8P38BsJAp1Ib$($#&@Aj^RmO(RMnyyMV*9v zrq4uvR$PMox%vUfvoRd?(cW%Ho{fpf*N9ojv+)+{tGK1;QfWEz_|_tiZv(nay(R8V zXQKk@r?$%Ia%n&0+&aj)Cx9z}x`H!a3Ji$*JM zVT^kRU8}g|F>WOqqqrZC_xlIBPI0B~OZU4Ix>IqLkmIT$@3&5jtB>wdT+>>?mfu4kD)%=?=vWTqiJ5|6%@YZG%vFf zy{qF`kGx|4KwhzB#`8SViY<@4Vh=!Gv0YII&GR7ij;h>Y^+cYvJJCG#-iPK(55>4AW85d`HTAARi=?k&+%GY1hlhE;P9xYIc}}Y$ z&vIOOAf6isLnj_q;eV|xMl*tU5j zJ+{M;kF7KEv2{n@ZvomS6RO;Rwv~n#HgG{y~zaZ^x5^*)34ke-Wi3u4@d$Vapqd3An^ zUevMugnVpUPfCxi8S=4pLO!;O(cT^l@-=oa@-_B$Q(?%XC9+3W9p0R?;X`T^VE0 z2g-OD^;5m4kgtEUk!NEedRw#c2J&otjC`iIpPas8?~86wl_toq*xis{u`feQwcqQJ zUlB*3hm~s}%QMj9Y;oQ{0y^?pw54ahWOU@otMIE3Pc^@$Qb+D6U3~I|Mzh zxP~$AIP`_$+Qzs}Xo}*x#JC>lOT`U{af8tlin}$&jX`S_HzCGNMN<{`T#S1OtyA1v zG44b3q~cb^xUbMxiu*0bZT$q-CdKWHe7w7%ZxmM*dHvN!Pbuz*7uejDRt|NL{ zap%UkZs=RZU4dNhTJ((Kh9mFyHuSyX#>cqHXu9I2$GD~FEyY!snqFTHM83Y9iniJ! zJd@8w4=A?}DxtRi$k&|VXsDjkW09{kQ&CC9%|P2ovyiVeuOeS*mZ0s_yAt{5qm5_> zwPl{8;3f`ViVzagU)Y(p2RAzJR>n*HBgUE=8WxuTV9${eY@V8~Vu zR8tw}paZ2Y$hm!y_j@loSiKLSI?}`#H$BF^ioB1v(FvNNuTWiO{D2OTHlk#FPpA7R zgAP@1W#sdbj}B8?6Lh%L0y(z>^7t-7J|9;gpN}!9o-)Rv`qG2Qxl@sI=ODi_y@oDm z8lDx4kzbiUL4IYb{!IEjYKZ(ix&R%a%6*W}`L*aswGBf)=VOu2`DEmCJ_j{W+yZoz zvb)x%(jJ9*TTES|FeEe&`rwT#b&Eu0_th z1vz&rYNXy7sIfFF#=RQjmZ0O*yAt`lm6*ZjuGO|ZY9f_J-bW?meH@H@-i}5-Z*5Tc zi}n1CT;w_Jftso9GUPcOf;^{VkmvLflxs{RU^l;=kJqNW_#)aq1X79lqaiv?ZO5V0r4x{IPemTzg~;clAM*LQ1D&Ca zv8a>u0CMgWc)RsE;ykKz*fQ$hl*Xb7!JU)%zm4Oqv(t-i~oA(B2GMT+O~Ru%2Fxh++C4#t0B+n(a3Y!4GmL9FEm`b3_15&tW#cyy=QTB5t8HpsbW zB9E^Z^7$x0J|Fj@vC4P|-78H*&Yg~&J0JOZ^j3(|^XPr#=h14E{G{|^`n^X7p$-M% zEHp-bMxBnH-6FJ|i;`C%w?}=TMzV()CalVK;(L( zkdJL5^0CcA?U+M zqmgqTM9y6l;&epcMP=skp0vhN7Ohq8?&vG2GIEt#$W@vm|J|k2(Km`a2YoAbLC)xl zoG}=!(SC1_x)1sJHW59q=i3zI=i6-L=i55evR-(m?DR_de5-=|d~1LzZ>i@X^7E}Z z+FNZM(Y{hwR7L8C_Lpuz)urc?=4?1jiH_9k?&GDe{~(ir62hmdn;Am_e_e2ra(>MG+?bcnPX zIrj(T+&vetiuFuB0C}~yM4p2)kmukMR8Lj1@athM4p2kUS<8MN?CNGQ~|kORpfegQ7iT4BhNuwbduUmLv5ta$hqB- za|@7<;|}CGn2AnN#*3((G!HrVZRFfl$hlu5&q1lzxXV?>PN=<97CCos~Pc@BO+zBc`V zRx}OQm#r41*QPSa*QN%@*QPU(?*;lGUz>)ZuG;S?%GD`l+o4x?JjmoI4OXcNFqWPDDNn zi%@@Myo;`qK0wZ0jhy=na_*LIre|RfG*B7)qCrwMm3T zr=sgrWd^!InuT2NRpfe0&|vkhM4p3<=ti|=7ISAQl|atj1vz&=$k(CC z$k(BH=q2rK5%P8DT{Ks1pQD$h@6jt#wfE8)uZKM2Ezqm#JsG_&wMQQPdB~&hh2B)} z)yOkG5-nESo#<`pUgX?K$hj{d&)R#)GrkeMuZ+z5+=WXekaKrI&fO0=w-)k@H$lsl zaU%LyIvF{)Gji?_j=oph)#yiQNbH@2{7Ui$@)`Ub`3(MsF5fCt&Rd$E!R^o$YO9F)OV!Z;sU9kj znxKJFf8^)UFyym34qd0-htUnvB;*;IfjmR=&`|ZhjeJ&Dqv2}%8r>p&kDQzNAk8g< zd}eAOpVgM=Hf6L$qooeWx#uJA_j2Uifyig|E_9DF?nmRKhmmuqA?JRCd{);YpVjiq z(mAMxJO>TXcvWeH9+sLS*E1O2IyODDrM?Q{u$aC-o znx>5P=qc$(4>fP?6bPo1HFR866dRaOEIkz5iZgb@0I2(Bmu0!*cF$^t~ zZb8l+i=6uea_$V|Id~f_QpN}9P3dFg+;zygyDm@XU|-}pXpKAvoss9D4|-Qsu0%_u z0_1u(A=kSTEmQBq$m`%)^s(CJpcT?9$hq$z=dMORjz5s+VDFFlw_{~gLu;g3$hk)# z=e9)7Jq39VdZ4eAaVh#*x)M2eFmmn_$m`$*MPPK zQbXh`QaiM7UbrH4LVistK%UbP$a6X#?XS3rsJb)-d3?_!k8dHWt={*M=X5PPRBhj( z!=;~*bMrn)b9Y0YmjjXKv?DrN8E2tmrSp+1ULT@6&W2yCKhbedHNxhCD-OpbpAC7o9G3MXquw@<<1v&gvb3JVWEr zxoVq;E|8`m=RS{|`zG?}S0m3*na^lbMn%+J+6y_iCUR~|)Jwf>QD3P;j5|NZU5>nu zfygs-7rH_j_oM#O!^r!XhP;ngkZ0&E6??ImNC(!MAVeL*wqowDON3{@nRPUoZ)%!W}jBh~qsO?WQR?1tM=9Wgz zt%5uoM3Q(QslWm2zl=7qfVOphRAb&BJ$jiK^+RhJ|-j2{US71`*;s| z?mt5F)V2=ImwrXBO53jHU1F&MS|n9Ni=|_c&s!Vh^L9R3s^0EsnbZq;{;ozoqG4!- zddDE2x5vJ+W3er2sxt}2C{(wAx+kKhN-+^dP zWgLnsOGhB*HbKtqf~u&u2dXOdiE#sC+$iLA{|Ks~xTn#q8etCRqgrZv105v2gOZV= zWTeP*|0nX?Z?l$vr)w6f?1((~m5}GYEAqM@fIRoNqx#y%Sme2%f;{)Lk>~y$bcEuT zp(CXg$fNoSc~rlk2I?)jE}i?`(NSvK2Q`%TN6tM2IkyS&+;>7=y93eD${39DrJIm* z??ldh79FGBIp|pFl^FLnj+!X<12kW~U!!Jf z`w^WW{eoP%ba_$6_ob%AD zvAl)6URI(W%2YPbFpm^xZS~M7 zwH=Laml`A2I|;eoxgkzRc|Gc<`5TGuRK_^;f_kT-v1*%v?vrLA*LxMY-g5MSdeAvG`F8?@=-?tF5Wn_oQo|zdTE3+sGKT1SWNyv(lknF8Uh*HQX zyQpMjWoBnnRw7ybUg!Nj-|N5UFF-!+%h7C|_BF_-eJhH;9$k%2dv4^@-W2(? zcSb(#?*xrUKJBxSPy2GTT;|Wwb^=O6`$zyCLTeLcZp+kSo3j{j7{N$Q9p?HmPkd+AJMJ9`7vj zc(>42^`^KMy87Mm`7A zkk7#altLNHP)cbfa_-N_xd+f7-SL-DD#hJKYjyXey}|o1wdFu=D~{5ttsF`xRYD%Q z4)Vy)qKxY8h@9IS-LJOyLfa&iMQzj31JW$y@fIPE_XEnM-W|x}9Y=Z9b}6(a{>$fh zwWUA>q%_FmWk(*bIPx9P1o`~6MunBp8TtJ6MMc&24k{tNk38OZ`MV!|kne@} zkne?2Xshmp@yPeWY~*|49P+FEUF3VA)UD`VXpej^^a&b)d@p>2d@n3Q&ugXMq8Fvr z$ZOt&yyhdw_rhi5d*L6{MHxwNNB2TD^s3tOpx31W$m5kp9qsC zuha)Q_kHBtsmLc{HS&qr6pK^FUgYa>7QL&sOK7n4cPvi#OyZqry!+8m^*)H4TNaH_ zTV*s#s*ar75P25!j#`O!JG6+suIlE}G_BIh?tJ9jJ?OT2521v6 zV^8YeL)^s>mzaBuS->wTN$-c9;rTZ?$gM*y-+^& z4nPH@cSGFh5H}0?3@t+6>Gk7B^sq9vpd!)^(t#weh#Hhk{CaA z#Zf7ZQVx}ss-abSy=;RjtE~g7B6SPB%aKp)R@6xAJC6M6?%{h9y4B*CDh?aoLh4CbZLR525x_G4z5|8ojN!YRLO-jLvGjHmHfz31w;!J9S-= zcis>Ae%pw=`oqZgTQYv3CgVM^jEv|T%~2isgf&4vVK1P}itB>1NUtJag?`AZ8ipQF z??mJiwixA9+X|FhT8o^!1v&RT@(KG3HPDWdCQnRwNE!E`f>L_q++4`Hk09q(Mb2%E ziYcQ7Dk-%>&h3Pp+XoHOY@<+F#eIaYgQt-=1&drV< zRmOv;np6-uw=8mQ9prQ04EYp4hw|%MwnM&_uOVN{kC6Xvz!LPm&h9?s-$J~F`e^m{ zrr=uY3}r$7t*yLhqcVyh|JGJ%G(vGTLRvV???WP(mcr5vM_3< zYgq#MT2@28mR*tmMSB$T9XcC%Z=2A5&9)PHZzquVb`^PV_wuW32NjnV9g;F3uP-n1 z-pV4M(|YKL;##1!Ixn5j?`rFYj!SPKk1`N>lyS(T%tNOXw+wldpV3*hZA0g!UC5&x zLmuSPLg^+VABj52gQGMON&Cv~IJcn*c?T~X{LCzhB zoI4cNQ|=U$m?HKh`~)SH<{;-TM$Y{SdFKbvy^1@7)~bI0qEu>&{Tg8!DH-x88IebM z2>FzhK_0IzN~etG$m4ZHnbh_Q$}Dw99`9}B@n#^Oh=r)VcJu?vu8hqnr?ee8_YiXK zIh0qu*O9M6srs){>~*6J$!jjF5d7OEk|ewovwq(L4fFN$9U6hWwmtjQpIO zg8Xz`fc$h^iw0?q4aiT&UFd{%^n1|X$Q4SOF{)5DG*r2H&@ib0@_tJrue3JmtnnJ6 zX?kU9jn1pBbLf2?eWcz&p?74gO=t2`G(vNHjy{wYAkVfc)~2{^$Qj4cXvLjFW2B47 z8Mlx#Qt+$B59C!KIp9KGBju$@~Qh64b-Xo6#3LGMm}}_BA>dnnWIyeAI;aiMUhWk zRpk0LLathS^p)aXLJOr=kXPCVd8O~8uhlyq`3XD^Em7M-v`kuxoVyM=cQ^7bk0S5r z9GapXT}0l|UF01V%M!icXo9@A4#;=ZFtkInjX~bqEabf{LEhWXXqV!)p*_+r_qi714AQ(FmiP%4W&N)6;u8lj`=ZG~K+Zs>&C-bAOQKFGQ6 zBj-*D3&?V`6uxyO)m&!T!dL;s>{%1Fq@`wb}>a&AWC+=q~_ zOm*b?H9@Z53+Rq=yP(9Bv3DM?B9GD!d6Z!&sd^_O@B9mtN^M`G)Y3P|xj!K1?nPex z5#*g;LK&5D9c7VjA?K!eAj-{woSPqc=jBiiWmG}Ar5eb&jgaSUhrILd$UA=rdFK;R ze&v3Q3Q3(A9>ya$n#c2k0`e~svy-y z9;F%bC>@Yr5qqP`ihCP9Dh)=?7=xTK8+pGAk?+A3=u6#$Ymx84ZOHfFZ502zZ#fbZ zw(3=*0`lj#=a4_YbwUF*Z%^dUZv)XjWei9D{5A#+QrssY?hCYEaZ5woD)f%xHix*~ z=r_e332~>;yNdfe#N9;)6qh1rG;caISaG?L*Y_~;cx6IdCG?)+>V>!`(Ls&ZI>fzz zhA6Iki0g+ADQ-xJ`w+daxQ{~IXXvov7KOM?=xxPi%N0HAYN1hTdjk0h*9Q5#^g??z z$_(U_vjmM(#!56^`VRRV{DOQAP9oo-H_#--{f8z?NpnXT8IUsyAkS7B`L+9TG+h}D z(F~~>@@#F9bGsqmp#zZrvfC8olQNgdE(^$tUOw9+x?sM@BZ9*X-B z^^?vArOzL=l@Dqi)F1U%#&G0UoN4HUM*b9?lI9}UU@3A9en$SwWWOTs{18h0MC`Tk zcjTR4LEd?#0@2PJBcI*Zk#{)|UDa%JkazhD@{TSAWqc@VdnD+&pnfR6Bjoi>KwjSr zlvC@Qjl904D1K#7{K^!J)>jbyr&X0j@hgMkR|X}fitTq8ieDLYkJ_fA_?1END-)Ey zP}EjFsC7_(6yFD`r)x9~rO?QqqLk8H6u&Ykeq~Vn%Aoj_LB)J!Q2ffE_?3A$+IeH- zE7KNb&>Sx#@BBmLoqvQfsdpXn&Nrd^)wUOT=eLn}p09AEdO_WS#s{qoI*43@bI3dT z8+k`}(WBbYJw>7&WkTN3v&h%78}g3EAg}%tlv6AH33>H@BCj-K(MXR3wG8STG&5*D z@=A9guk;}DN>8DNTIqS@+TTQ8X|ZC_dy87=i0NYhy1ao?~yUS92Uji`07_d5*hiiQ1BtOiWlRr9z$~EAkvgkmo3kmMN|p@*E9g zy_(}Gv|PRIkmu-uR;sNpS|trcp5sI0IX*$2V=h{wxTVN*tdI3-j!o!0_3lNU<2+iY zwkzld={oWp_mqm}$bvjaF7%_~iXhKXG1jX&s-q3+ZHPQaJG4=4ozNz!EAkv~A@5@Z z@*ES<5Z!sxkngBV}3s9ZaL-My_djl;pnHTB@R?QcdLX znjnwY1>LLMUg(r6IRTxK79-E`J-VW{J;*;ZI){>LV zuZOslk1(=UkRGMIFE&Rm^p)C*p<2qVggkOXJ{`}(CDD4=+`G=Rhxl)Wfme|nM25*#jm2jbj`C?h+Z$NAirMLMK!g8 zr;%T`+M%1u=z{$E_ByJqxPc*VIJ%{{@gZ&+dR%c|gt(>Xw&K=?xSvp6#qADpN6;O` zoego9P(8)n4RI+da)&A|Bl7wlKn)c4F!Daip#K#2Xo#zY8Y=F|5Z4+d6pvN1V~BeN zHBwx^5H|!RDsFU$n}ixG?z0fL2qjV6$`JP(s-n2t$p12$r&4tHltX^^)eyB&Tyx}4 zsU49&rS?SrlsX)>Rooc#KWQTJy*~^2`LYPLQ|}t&PpLamd$sM4wMmDObI&8^{)1jn zZ_>)q=lE>sMYZLLwMhk#b4w%VJ{fB(s^`BUIvG*rD~kas>4y`=FzN8b5b zMvzLUQ2G| zwG>4I)mssH9}UsFYI_R3C$&V*eGxhL-B?@E*ghsB@8b*PeJn@b$4)d{qwGf?NQaTf zJC8hG%BuW3j(XFhu~KH_xO~WQWl*Yyv3V;XKc(s-Kc%{&__qX*UrA;oKfMgId?H~?oX(ZG7iVur1Qu>AG(81C^uGEJexGyAJuv>_I8jb_m@k{f?Y_5ji*AW3*}B0?7AAdF1-lMy}tJ$o2al za{XRKX*6;#b044CPZ>Jybwygq+(7Ik$JLO*Qxsxdt}N{b?|v>mFY-Y&=^_dp(b2&%2#(a0lzit4KEb5u`SfIRXlk9<#$L2VT`Io2zEj6BB|$a8#) z+Nt+PY?sZBjlB~LSAVn^oDwSAXjK8>Z!I-sJAp8Id>*f zYk^KGu0x3HhB7FwcZho%omSk)5H}fRRNSW_?sIfTamz#8dX!0Vzl6A5=$zt?hqyn{ z{ffID;_je76qmYDwBIZ!v*Pk1@3#>8Q*q@(Tn&^(aScLTGjvgL?Lu4^lvQy(LR?>T zS#iTd+<25tannQG9CSr-OGDgR^nl_vhPdtMs^X4>xU(p`;{FP8|DtP(OVK#mZ$^|u zaXFFqn;%_QT$vE}D9Wk0Iw7tRx}mt%A+96JrMTBZTrYG}aYI7fXp~!VQ$yTLbW3rI zLflG}M{z%fxGm_8;tqtklPIs^E{3>k=s(3JZ4&J_EqYLK*^u{}2PLG5-CHF>Tt$>m zaWz9+XLLa0eUAK`T#Nji+=kL_fPV2eR0=*%%Mmf~m8Tr}SALUltV3b!HhCK2_}>NEv+vLa-APWu5&3=NxfB(E8Yw}ss?-K~Z(Wf0)(h28?_lJbPDVA= zHUrg`W+Uf*jhy>ytWEnpAC&B=s4W+AHHsqFpgeL7YM`yE!Q;p^Xn|aVDabWgfc$gs zZ%{L>RBXSL%MXo_NbXU$aRoMehR&%-ge0MTMyJnZGBNcX&`d$hse3}kngu2V!f*Fk)VHrGCUJiHxKgt zRvfv`6_L+cO>|Lpu8Vxuo-pxz20RyIS7{Q- z4U^I!?=3s>-U^`+>Me&{(|Ty6+M1xzQgh_o_Q<*K#M)GisX@zvwj!UkgUB^Fja-Av zD6vuOnR5-f2Fad{YEU1!25ph=w~lC>*762&4JIJh;A1piy`B4t3%&5fK}J=UfgJQLJCXgG2WCL`Bi7V`b}70S>k zR)fXJHCT^)4pKfB)gU+WIVgc#Ec#r% zGm!7Mx#$bEEe&m(kzezU2AxJP6}Eai_=xu8gw=&_Mf}>Dx^ieFLMSJLB2ANAfL|aD1W2a zm8p$qP1#kg4RjRk#pOlc&%b>zC(kS1#LyH%Rv;sLs9$=MU@)q4n^@h^nXze z>ZAA_isE-D+M%_)f#P>4at%I4yVd&zir=AVpW1#xze$^s_puLoA1Bda^gN7s5U^0r|p(uWbqWX<=hobl$isE-@`=|!FQTz@? z7qyo1D1L_`*Ptbe-=XL)#l3;9O1+U+{~q${$D-@%oq^(aDEe1zOGDdc6u(1*PNO@D zyNcp>=<~cES6f~j-6yp|&h3Pp`woiVp+Spd zalS)?&ITpC5M70|D1L_q6+!Vk6vgjQ^jss|p(uWbqWB$(;&&*D-=QeG*1Q4v${a(! zGG|dP^sP&H{e^6Gy=Uj47AhI)@8pU!K@C-F9_rQVbsqq^imb=6h?)t3q*k6a#kgZ*uHgaw=U|vfYCVg-P+L3nrSu~5&buS;{B5*Ay(5q- z{s~&7wz+7r^d)lc3gp}au{Krcub@<~L~RcRRYk7Q6Ufzg8o3(n&>B_aMdWI{iCm2p z$ko_}T#X}Wl~#2cxf&_DMb$`;)~GiZay3e$@6}cjt&^%E@2COtj-Ek3s<#7jHF~2B zYI_@Plm;W`jzRG=6l+s8HU}LKx)YT7)uVWR zE;G?@TGf2y`)vt2ptkQr+eUOyZF@r7VRT4s=R?~K7+}@xwnyXv%V2MMT-U1 zMZPPZ59%K@IcO2`Gh!|BUGWR@UGY2eE-xaV&bugs){?wObUJe&pU(U!lX^=bpVQhX zv)Z0OS*51P`)Gr_k1i;OdV3+)?*o)uZDUbhX%ceoY~NMn$xbg2;Dw zMdbR`L#|qDR9tZ{ppw!{$UA=%dFSt-vg#d$T(w!KyxP7%6{Us9xvP z(I8#H!{{-^okwf+%(;zfs_mZMynmKbAdiw6d6b8d_f{5pZ>??X4#A-kwC>TR-Hd z{0!vlwiJ1nKcZ%u?HA--?m|zg?E;#h^~L@RS1S|pw@>mT*PsG=S{c>QGg2+&)i*_6 zeH+wLyHzet+TKI0q~XZ9laO;4BR@elpw^1pirPxQB4->$&PeD(n<}0osC-bf zpsqoE(f>5wP&7)p6Oq54GY@&^Yms;UD|%iTzo8D&5#+V}fxMO*sH1w5^^JD^0P3Q) zyr`@65OQuA#u4)mJl&4=EQ3L&qc z9P$btLvO0LG4geL9`#mRXY`iT4LP?ja_*E^n^ymI&@VwJgKh`i*Pnk$*Lc~GuR=lO z9hF1=IZb`!@9DQe1C-GL4fLvFZQ6NHw_tDqL_t84!`zYmr z=*e-PEc#)6VVm<0m2n8|(a0yzKIts-9M_QNNc|2!C00fjbWm}*&>^WvtXH!= zijJzS7CI)?LmsaM@^~*H?{^ULekY*cmGKceF3m*FU4Wds4ta0;k@t2XwB7$MpOm!H z66mBxc?6x7DkG0q7kRw4$b0LHd=keXpTs%noN~WJ=cTWa&&FEhv#|yJq2B$-C-G18 zr`oQf3(^hb++>5J+=9sG>an2KLA`>;2F*m5G~NQ_bF~`zTgMiI`a375{4uu6jeql)L*A51Nu*Kd6DCapkj(E7vid*g!Hl58ics!sJPr}8alfKciaQkIPN1ZU`zyrVMx_;( z{QYRZX;5;-RT>(-S~W)TceE&jM(K$BUavp$JK7JC z@4T7le#Om0S){L!&+ba(v->m3s@~nGyYBsiD7)IuBHxX-Q4Y1;GmP&tNhy%W%Zxl; zK9oznC6QktYN6a}Yk=}dO^|b2BjXFEGo{oxYEuSIR`FrF#Z$zcky9c@A=TKR-T}F>c z*N}Iav~=0vV;DO5pi6;MU#QRLkE$Q9}oYg0Ae37Q(TENCloH4Y+w);Nt^ zjmv0`s&Ngu8p%dP)u@kLjkYNM?}MnCR`mvQH6|ce<6~4qy_fgQq_)(kj+6m8H#c(bqp>zsCF$kkYd{0{Ud zv`p35j9iTa$YRpO_53WK@)wU72E{9MvwVgmuN@tPByM{bol2LqnTfOO!@4@`2rP_+1=cJOzxsM{} z)<>$_&nr_FG3yFy9T-TJJCyO+mE_PhmrSt9(lk2pswmoIwq=fHq=dRdC+T8 z0p#4$$hl2oZK_L`pm%~k4q6_x1Gz3okWb<{BJQ z+{n3)#@c*e2E7_IJZMhP8sxfcL9WYQfMdvuaan}+RlcyOK6zdZilveCh)#k zZRwFKSpa>YwxVc+R0?_Cs>t&;h)KQAAm7O^qmgRsjz&qnkaGtk=Z-<6)jJ*ePM(9t zsBLj*TY<)^Z9{1L9r+604$3+)YAX}e0QtJLM80k>AfM+~&^2AR?#S2e9pvk_3Hdx9 zLB4Jm(KxN;I`VZ(F)6xk>Ct%g=0d)LrO`ySRYa4ds>s!7fLx7d&}8*?K)wpS(G<14 zjiyP1k#ol&=Pr!3={#=^Iv#W_kk*m=Nxf;Wet1%t< z?*0tT)2fyrS7Q(IDL#z8RPTA@YTQL%t1a0SzLO=TLf%nUSDXay1@DtJKyI zt&y4`=e9-8?H6lPH6{fu4B8xY9QkZqMy|#!6 zK|gC%DYbyV^>j zozf%7qtrwmr3Lcd+9B_)3o4|&y^6fIw^97uqxiW;-rH%kM{`_7-dnPdqP=BC-dka` z->X8uc~vOBDimK8^4_|jLyGH#ytiTKu-ZnWBhm!qQ9eZ;Wf}6`zDM5MW>i6Y+m5`q zqsV(J{Bg9mYRLC=V{}JzJdM1!mymx}&>MMgBhY_}8;24z#O{>I$g7%zysE`0nR>rN zKD)b6O0^w8simXHxql+({)^J8H^p>*<3`GgGDro{{n8^St5h3hlb%8kNGH%pot%4S z@X1DPsZd@i9rC<6k>@Rm3Mr!w`cEgUG4lV%*A3-Y?whE9)CYN#_mM}LiXPS|pQ2kD zWf>}@jMb>Hv<^9U8*=XN$hkL=SCH^Yw1O1KzagC!71byO&}Pk15fxWkbyQNSjXXy) zAAkaqU7}7xb9odW5*XsDa{!hq&>mrsAfDxD}|D;!=Matu#NXrMSAt zD}5RHr!&1!Ph|{2Ug>D`gfgZguXHBrrMN{QZY64{xF18@7SvmD2SVIQ)JSm`L)c=dGxN- z2|1$&a>ihEQyJsX4QV>^N*ALc%3XnmN^6lv*@8UELF7B{0vf5fztI@!U*wGBbE1qa z$SW<3CMm8Inj$@doKX{bjwg{<+8RyO-P0cV?)e<~_2W0>ivNY~)sx{i@{ZDf#`jb; zM^q;bd@pCD%}Lf+BJ zSe$Bq2&K@PPb07SCc2?f66f+wSt&X4c$tvL%a6R~^5~}Gs>a%+n#dVVkTd>=yyn+q zQe1CzN9vE9F&sH#D)O4=BCom9yr|C4BCq)sbXTMFK>tZ^A&)l%dAupeYhHvBGV15M zC{bFCoUsWx;}G(iuc9Q1yNT|R{zJ}4{dttp3f)z8pGSUm8Gw>1<2{s28jhSh33;}! zk@tHXdB2^$i1s@bxh>E9sI3HYTWyq5*RmOMU0y`VHCtD7ukv9S?_dk?M8Oguo`zq3X$hp~&bIT&HvYdm0s${y<*A4dfN1SQynHJ1U{L2T@6>AaX`opD`RO) zGjbtk6h~ftbyQh#bx;-Q3FM5H$Qdsouf99ls_W4U`FgBI-q9~8nXbZKM{{IC-p51eam5uwb)?eB8P$+i)fjmn?NMFDy@cvXuOMgiLCzSCypJi! zueF~euXzdbntw+1HOe;BK-z^o-ZA9yE+elw$ztA1DJ~^yD5XWt$bp_k&Ytg-VSX^S4C~P(IJiU5IQOqK_0IH@_0>< zuR@!kPAH*i>>XZL6n`~EK6M`(Tj>(iDoHo zFFGuh{Elz4Xx{Q@hExUlYSlx&S}&k#T5}&XOL2qHXVMVljB&^r^U;gS_zuli+y?ZO zv;{e1KXS$&=ozi_D%z#oGV@+d=* zSNbXPDBmHEvI}ie?g6w#I*L5XpU9*9hkOpwtdG8Zmm{b!dQ>$ihuTTiP%YhWwUF<( zr;zWreklI!S>$v7HA=WYc5kghKE)f*evP*ab<`F79la!7M?T|4e~8pBXczJ+{vCPs zmxAsDrTQ@%IU`D<734vkHAgYz*&ailt!L2upaMTdvz0-ftwvDepw_69M(&I}@&M$K zmjs;+dTv8Ba%bd`dj-80G!}Whxya)k3d-5;E_F60%IL0&gg&(S~VGV&aEk>^OgB~o_eYgq_+j*=+3=6Do! z)g1Ma=V*&O$GD*6zeID~k32`dpi;p_kDyX(V#hz7E=kmZsvChf+uv zk?-zX$aizon7y-^Y;e->xYBm%30}?Q$;i>*W^Y*UNqAXy({G zeH!`o@-li&88?w%FB7-%4N1kNM~=&mURPWJ3}uu=?fC7N908QcdLS^V6)h4MUkIKRgj-ZZz1n@B=UZzqfwgeGvxgqL*DOYzJ+Zenjp{7 z8hMU4kk9ELaLD*b^x-VNmO?mG}wEiX#0 zxZ=oFtBz8ttq!_RdIEWrmdK;Lj9j(1kgGNrxoRU(2UTqxa@A&`c-2t6YA9Z{gM42= zvt>c?s-bw*P`qj=gW~$4jM6|9uNsP14aKX5;#EVLl(7ZHtA;YG?KH|N{ej|DL-DE| z3ROed6;~X^s}_2jqa5mOA9~+FdDYej<&)k5lMeQEge!<5CXf@g7DVuL|We(eP~=gjAy@5l z-!#g zeFxD(y&7CaZ!7L58m4y{_a5Ulz^~`%9rflwu4G9xL~Z5KP^k*?9QBarcn-Oeuc6_J z>yKQ?QD~&vCZbW&G~`j{A&;^OE!J9gqj8Emh$cwKkTWhIXC(a2->vZKk!$||a_#e@ znX3K6$hEJCT>CcY7d`dfM)B7u6n~9Ez8*)=46Wca^7Y7jJh~pGkgrE=G*fX;pjlE= z*7?NV{aqxdJUbS6BDW$8( zD@ZsStsp&0t=_CCjZ^@6l=3K@+Nz=qQcdJhnjr7ve<+iBJEHrgH;_kp4`o%`2$W44 zgFMP~Ha!TuwN7;>XtL-4lBOOB?@Km5~Y}&sG+BwmQhOHA0Ult`+jhc?o&80myTVL*CI`R9P7d z(4*24lgGcs;i9A$g@pH-rG{-*)}21b{I8O#!1vfI)^;lb>!Jn{u#}dHRwTfM^B%E$WNd0 z$WNc4$j_XQk)O6-p%Qx4U55PXas~Nmn|L95+Ga*26_*p0k{(1}RSD!()kA*TwnBc| zc0&HS=&Pu-a{D1yV?HXQw&kHM;UfS3;r9{9HO(1R3GLTeYm9!ArlIN5EcCr{*P`#F zUr|}j@f#{99YJ3GAIPguekpo2D1#nRTqRUqs)n5L1ad}OM9*YFKR6(k=JtHmFVZdSYYzE?}4*K~iBL%u(1Bj2mTke|ELk*m7^E!FzI zL9Xsb)8GyEE zRYQ>LG8wrp^N_2t8f{nH4``>f5qTebkoR#M?N;w4!IwqscmxVKoFP$;No zP@ABak*~sALGL49kFm(t<0CXh*JCE~_4pe3di;xgJ<{Hcu19WkNoy{Md_5{6UynM- z`+XYu%5*`0E8}%^UFwOv-*=JsI~x6?-j9*5%o5~^|BP;{cN_Xo+J!vIG2~J1#@cjF zv)qbQBB)MKbL0v=k6f2mkn7S5EmB?jA=hODa$UYe{?xV|xk4wAPwd|)(*v%1Jf zY8z2e&9(!%YNwFT)m7w`-g7sqT2@p_8M#pzDL?W`OChhcDk`hq2FO)whg_>3sG@rN zqAJorJ=wQ&*pv3>8{iZ|SZ!YBh7D7MkTopq;SC1l}tGAF((@5mH zOh@&#s?U(?vK+ZC80CA@3teLXw0g>P?4SjfarWP-W!0G)7M< zt_5l?wL%`R6Y_X{&=fu4h9jS$>F60{%t0-sFOYM;Mb7;d`3ZL+)~o8KNQ{&}s9I1% zTV!cBXv?%s<=#O zq?8SL9}mTP^$DX48m->y$kk|peCoO&*JS`2tGM^jcxgEDc$1LF`vOhXsauJB>b9YY z%GiS@Ne7T~&miX}Nfw>DT*%d}81z(7x1eD`laT8?2f5CRkn6k>rG6^*Zk#o-@=cc$fs!QIWia||-UJB}oT#Z4<)%XDAQ8h**SK|}pYMetpb+^zr zT1&DN(b>p`lIh>s^P}lHLv_);YI_c?QpWRWjnoNwA3c!w@jgnSj5+8B#eIo3NM9po ztVPb)iIOYh5b~8ijY?`Ce<1JU7V&P_}e<>o^^<5h!N2E7q9GU)T5Z;`L+ zdgQD63#zVa>_o1{@5t3ieP5FJH^uU!X}Uw}BL5WPW#n&)^+Fl4$L^FN$e)-;qgl$B ziu}oX1xl^BV<@w99{GR0{2L9@(=mDKBnes6n;xyzdC7;es;w}}CY3<0P!;6rc17`T zx1rfu!CVyob{mR+yA3VTD5sIvd>JiP#!cikC#Fe~u+(coj?0dgD6RlYA4gY8=!#XmHSspl{LJ%KZuX?%sub#*ZMM@w4blo$(9EXZ#L| z-|^|AtMC-^Q|eW8UaNl#`6=}wx}di4=%O?Qd2gQ~@9i7pr_^qAMR5nwRp}UV#sw6w z2J%zt-VB^}tu!_AN^>Hw^jYNd(hd2%3_!Ov@(|?nG8y^2e1h((_bcR+vkoQZh~29j zQ4(n@@~RFXuj&j+uHL_q&qj)j>`iUy(0$VV$hi+9=hlq1>B_VU>K-&a=+mGzL3@y| z>Ivj)c@gJ#eEv$zC=GL?wb&|20gC0Uqako^rPaA zg}5`Qj^eI|xP;6}5`I!#D&+kZMpYHp8TsrEKt8)8Q4{q}L_WJqkk9T9$R~C)YNoh- z$j`6~$j`99(NpS8%o3f`%;;&g(y!B5cGS{?Vwayqo-6hip|)M2?I0Sd zwzHw_F7kQF^gyJCgQ^BS8T2d~qft5{KmYomacUccCP+h&&&D|9v+)UFRwL`DB+wpQ!CoQ zy+@F%@fSL)wtvxi=`M0L?#mlhqagDATNzzY++*mXR0lcZDdhUS9&1zmh6T+IS{<}6 z=oE7Ot|Hej*@Mw8b0F_BAG)l0OCs;GD)KICqrcSK40)F?qHAjFimpqqBd>V?@|r)2 zwQ0>ugSH2q2}+VLic62Y=G@3@E{uH5%c8Ej<`t2zc|GK7J`DK|n1*g^RdbMcv>f^Q z_Z_;U-d{rRY2?4veg#d}C&PR5CyD>n_7cc{OTG$Pr5(M7&PYShFUpvT5^~0#Lo-nl z=`-ZIEJ3cz2IO7tM*h3($3xqNpnuR%?e{kFep45S_FEr$zipAvcsG<;^Y%nO0%%X4(fymH?|9i-vNqf9~`9qfZIw|89^pdm_Irj*1?iJ+R6h)#Hq(@#sHsrs^R|s{{DCLmZc=mP zIoc!7(I5FQ@(o2kFJsXUIxmxu&&yoo^YRPwC(;vWgYL_Fi$mWbPnxX01ZwKW6 zgW`2mSI@t`$p4STd&tkV(a2AvspwCwU{Z--XrGiG z`6T8-K8dA}f6Jf_I;pihh5Wk(tCDV;$c?5PqjTB+8UvY zYHJnR3YJQeuv4QnM}MjJIdn~GhrE_oQ2ee%OSG0TXoqrV1br2>Drgh(?>X#4euX%W ze8>NZ(zb}bd$@vpza^HA?)YLT{xc}FSF?3O{u$Kg$lu&sguc_7S0dMCEArlsqFvzSnpUp>-6jv_9RY5Zq*C50- zM=2H8HpIP%K2=y<>e0Y3Zj+DD<9;2gpw++evsD`tx{fxAg?=0 zro8?^-c&S8dDoEtlFpyVf4}4z>fIo85?>;p-PA?GpPQ9MepPOO{5N_AAin~SMgEJ- zbCEkQMLqbXhiH*DAb)pq9A(p0-9+j1JbHn8DKAn{u0V>8{LD*?{LD*(UZ|n~>aV=w zXn<4(xuQC9MME@Lt6v~r!8DX!T{faMdR2ac?kX0ySd`HJyohT~jD|_ckvnHb?pzYN zq8b{ZyavcKI-~5``Ekf?RE)>(XY)Aj;T27&{%DaHZOIMI99w1k| zKohkZqj>lVN+R#FA@VLKB46_gG+7nv&=hGCa_4=>oi8C*{E4P2?>X`=-zgC#QV#7h zEAlQ&q8Z97k7i0AB6qHf+_@QwtJmUgXpZu}LvyA6$Q9osS4>0mw7MAi3N9h<@&)oP z3zZCC^T%kuD(a#IQX}Nft&uzTLEhzPv`~3dk!P$${yR^TQnW;QsnJp?19HXt$Q32gGOboYzJfl;yPSr+%QMK={0J>q#WS=*dWqaQ zX6bO}w8#~C&`RZ%K%P+t<l#&?@B(Mn6d-kULLCzUH~;o_4tbtybPPv_{&E zTyY$^;u2b`)qBWSkhV;CmnD&R*&X?s$D(zrn2gp-(~&zbLGHW*x#Af5S$P+cXFNr@ zw9D9K!@Eq2HYhI(+9>5j?pzePb9r=MyKI0qDep7%i}X2iMQ7xSK4`O6M?Q2v@v=T#*(X(rPZ`D`<|q%Rb1v+=P71C(&V5oI^*Y z%gEPqAGven^5Kd^=(zIIAkQd*@@SXUk$2e~omAeJ=#_8W_dIb3jB3B6SGA;5h8zJwq3%aC=p6IgF7rFB&cdHO0e5Bag&fxOF?$k+UCC7uV0g?!ERP$d3~7O4q}EVV%H+zGk!5afzUD2nptBG1^2@@tpJk#~6$MOEIPD4O&b zx$|q}&atZU-II2i21Qq1W)wrpfm~4pxuOD!snwduS1<&5m-CPx%ge~u{11wyiq|N% z6s20Yb3)|KS&=IWqd3Ydk38d3R6x7zjJ(SsD8BMWqXg1;;SVirm%%}4IM8oBeY$Q5VNd&;|k zJmVEAs9nDEQFxbGQEKJoL20A{$eqg~cdmvWXqU}UTIIDy>7@3^6+MtEhMQifvcR{`D8YAxispQC14{TkhndZMnX9g1AL z0aeuMUgX-tsGIW6A>V@s=o`iUhq_D8k#`=wMtJ8bkh^3-o|^}KtBMlHbK4=$?SWj; z5A{&qXymzb(07V0Mt!6e$a8-|zT-!byPQX!dkytd#UtdoX={e(=0>h4hz2UJ9P-?{ zXs}|RqM=fALbq`iad7`a>aD?qw=E^?P= z$a7z#*{X|3-%>Ww^i1oGS&$X%8q&s~F-sbV|w zejg#veT7^RwGQ8ZC@&H6+{_^S(3VCjOv{iXMkmrs@ zI~1FUc1lx_=PpE^y9v3=KIFMa(QZ{-K%N_?UU+T_k z$a4#$)2b+sJhuz--2TWF!_e=_n}j@f3A&)zPw29=9(ljJkmsI8?s5%z?mhH}D*i#9 zo3%lBZXx7~lIXhfsvys8f^I7IIl3dYL7w{!^4#IbU4B5GI}_bg#WLi%7m)XR7rEju z^gwy9kmn|9$h&66lA|Y5YUH^&kmr^{?ot(bZY}gw6`vu`9fLe~26DxG^sn+(BhTH7 zUMhANy^>BK&%KO1_c3yp*T{3DH{yM)DiR~lEss36CUQjs^j3MTkmvS5QFxObEmA)e zRT_*ucRcdk1;}02AkX~;MN`FoMKzQctyy?(3RGXQbf|%p8F_Ag`GczEj0OC?$5~k zJ%HTh4D#H|Xp}1MBhO9sd3bJiXr$+ZK6lZ{#i`kmrs^ z<5V#VdG0af{a!+@xPiti?=kY+Xs!6}M6tMNl9UK}ZW`pd`H;JmMxI*<{h*54$a4oG z&mD(c@gw?Cc?*!|ZbDNP+kvJ_dy(g!M4o#Sxy#?kb6=nts)+GLcy1BoxfPKss-xM; zYlu9z1Dc~)S2R!RfjoBz^4uxNT^1tGU5OT`Vl(o7?<3EBj$H8;EmU5-*5SG7(Gtb7 zp`}u8=YE9Tr4jPn7HGLDIw8-Ug*Y8dQh7U(_j?-sq}WBYTDpon_aXA! zNMDA##7CZ+6s=K3M&!A*k>@r?p8F+QtGsT=bBCjKij76;T67t-~Xt!ejqCL_p@Ab*0k75NjiOUR#~y+!^6ElGQ>R9^#9 zqj)hwwK+mDefCuc`4hAj$aA|Qe}XmudFLZgeAP}yuHBD($4?{IUPOtMcNcl*uh2V+ zMeV>(Xh^Y;cb*h^=h>0FltrHVAxf%>y2x_}BhQ_HTrm};P~IZsxtmcc#de|8(thN* zr;z8~M(*+&d2aNM+-FrJMxI+9d2UVQiUug7@>(I!?SZl?)(>Tu1|!cMk34q)a+fX0 zbN8T}syKl>H&UnY+yuxKNl_lc^F5$T;kh|nX zo?8@ss)~xpbGsqW9f({p0)3{u$;fk;p%#j*L9L_>$aD7~@AoWnmxsu6pP@FYh|)DY zw*d0oa>x}Qq7KTdi#)dt>a5sTsGHOcdF}w@xs#B)EJvQZ4t=YNoyhxrj6C-Zaz*rR z{3Mq05+l#eiux#)2lbZqHbLXK+iY-MyN~@6P zZb6=V47tlK`OF9>VQ1AC-U6U$X(_m z&s~n@sbUlI+`Gv8{Rg??HCmv&INyfnrbUYs%Yv3jIg#fUMV?y?xl0S=xoy!hRdh$5 zy9jyidgO{NXod0)An*4QTBX*U^AkXcI z)+=u)^4w`?gJN^hCTS7!+;zxv_ak??j6C-i+N_Ev$a6FH4A0GnTu~HlQ(i^nxed`S z#hRf#QY+-SUn9>QjND~9^4$4opDI=(&%KJg-$%$5&(J~TMd=lun+zRMEDbs)WkjBv z7kO@ZnW-BxUBC{LP&Ja;+r-0h)Nz1uv4Johqspo;s* zb5r#W&&`fpkrzEyUMb|cHPKVW8ldM=6XdyVk>~bC?lKX1?sW806-$ukoa5KPkEb= z=N>@`6+49zOXra1-a?-H9Jx!vzTvsa(Yvb1j6An4^4yll6>U*6<#k7%I}*L8*m#s$ znv6VmKJwfR$X$*i@AnK!ql)Xua})Oq&rOe9kqxC+ULoYURZs@SYM@L~UF5kfkmq(o z?lKB_?nIPD6?2g1ogmSbUU6N{l==9rD})$X%);&#i^> zs^T-`xnq#$&Ool1j|wPnHS*lOsIX#(QBmmx^4!bFa~~sji8&xVHz6vniqyz+KSG|{ z7`dVaDy_Uu$a4pxvWksBA4p@6=T1kSy9&9>KIHu#MHN(W0eNnmf#JC+kSo%mO3KTP zJhvSBP_YkDRp}$-xs8zLc0lej2zl-(R9zKQkmnvi-tSrDip%I@<=sb~`v%oeEczh+ zPKOi+d2Vv#xjB)$e1JT+DypN3`p9#KBF~+KTrnNhRo)Whxm!^^#rB~3(gEbTXOQRK zMegzzd2YXgxqBt^4xu> znJRumo*QLIcy1!(isY!J@-idOErwbtRu+9BRYaa!3wdrU&SEeL2VR!joL|3hKA=RM4p=wxl3{6xgVhRs`vzXZeQfNqme5nqE5=2 zgFJUV>a5rn)J58fyx*h9bFU(I`4{<?#3bHUEV>y=2xhW@}iF9 zIW5IPzUHLJ*PI>snu{P`b4BE9u7Ui>G(xp>^`9YMeJA9rpMm_z-45h^oJ1`(_cHoi z`WyM0(~b)Nb(kC|;tG&=UIw*NMRnxg;%tt5H@YJK7UwYZh4RLr*3v}eUCu$?<)+Z8 z?$r_WrSeXpHqtrdid)DP&(IE4#2(GFQF)0FAF+krK_ORUM6M`|eAg-<-?f^^J8Fpd zc!}CCkayGzc}L$P?`SUK!wK@1AU*~_E0L#dMxOQ?@{VpJzKJC7A>!L1^aQ!$EpkP| z@54LFgm_y`UQWatSdC9@?dU-3&@t_3 zZzxZ-hY@4wass)_<NNsH2{t)sUZ|jgg~CeNXviakL-ol(YzPiHbTLaS-eC@CZIKJp^(qZAsW)vCz9 z7u^(%SF9D9AhkuV?T%bK9{FT1L=%;_0!@%#3FkxJZ)N0XV`DT$U0R^2QfuUXU6K3sN7J?XJ@S6%qM3>1P=)@XQ}Q30r`jkJ`7JdmCUTb~$X&7_pNIlzxcZeslXN0JKt2(* zP{fHq5hnuqL>xq$HRB}miMWA$U;c-DUt&(;nWwx2Xs7fp@>QirzN$QEmsU$4-sp0}kr@U(@qjVd&;(y2$F{Xy!(e&pW;_&r1GN9IHP9Q%&ZUwP8v%-JdD>Is)8M)9zDIfB* z(#X^5qAfZH?a&;}9UW*5^40G{lhx%2nj)PO6WehppM zw1?=X^a9t=Zt~!Bfm~oM1GyDk9-a~BcGC9$gh)Q&?EKx0sSpaL*CnBWzmisN zZRn(EhE81$luEVvP#UQ)a+mVRUFxBB>e3ozR9;7vRqBFV(GR&|0`lBxXr|`QL)SES z1Ine^ZRmYzH*%Nb$X%`@pTDQbCnDtn-tp*K-ba2cOCrBRu7muF)B*W9-5(XwjA5vV zG#Yu@ROD$ZkY6LVBTu`5JT1e*@U#-h&($i(e~+grDyr{_zD9nB+yhnBJLF-gwC0XM zWu=M8S1C(?f8E~k*Y+(teJDHex63CxNdD~xKY zwh~IBN3{{EqgZoPSNa0Ea~I^!Ly%ALRMc2`b5K)hA#%l9{R0)(9lC>tYuW=eTd^0YgcNltzeB0#TT0Yh%7pqzMbHSwQh+*ZYnr;`JUyyxya6>ih#5FHJ+fs>R4xwHf(&bQbw}bRYeoioelh=^y0UsLR8( z?;_u|EU2gMMm6NG@lDV))qajApf#{xU$D%)_$>^ap9raV~66AZe7d_VMN#tqwkvqQ)l>I0Ej)U?FqkpAR=(+R( za_3KwJAZ**YPAdU&U>Q&6dM}ECL-^A4tk^2mB`m}0D0$kk$3(Y`T7#C4yy`^6eskk z)QFk8|0lwU&p`YE0!2J zmJWI6xlu-~7Dv98y2v~4hP?9;$k#V3&}o!OwbxJ<=`PADJw(2~m&n(bY(4+Vqt#5v z(@G(Ct`(>k%AverD5o?A<&q{McbIQ^_&~FfUx8O5zmM1ytnNmY^a}hd@+YFq4y0Hm)NBdBD zt^S7mspM@`L9vIZqVxp0_APR4@=f9IfO4Tq$}4~>OGS|@Dk4`jK)zRPk*~Qo@->e~ zRa83#Rg-2ScUgwqWh44Xt9y~J`5daL*cDVqx`|x-H*#&XU&3E?Q=+=c%Zj$=4BeMv zsF7l2QDdnha+g}jU0NaU<16H^x;;@dJ<@%VAL%j3kMw!u_l}Q{AJsRglV-%&9DaP0 zqOTOoh^FKWT~$uhS+Sx)tQs1rN53)h{p*CjQQkMGhtv!C>W3j;{WLUAwHuND?;G1u zZ&e&dzNc4EU&U^te$tEl%vtlnAAe}{jYxOGHo-_1(dxD-S_5wYV-XK5A;_eDR z%gUjbTCIlMxh`_&*62U2c1G?z5Jid`nlS=Jmc}4=o{s$dJ%QqA^#XF|o5-F2L2R>vV<^E{MXv85=5vkUL*Q?wn{Z?-R9}3b}J;MB)5-{lIOtEQ;CVy#dQ zsV#Ds?#NvRph}@+Znx)mUXtq=xxl1!N zN3qsuuGAj6OAq8OW6(UU{($C7bCJ9Jj20-i6)lu@A$K{3+~p=(q}4ysV(DMxE^!a= zuj`5>MoXpS$XzlccPW9EY4roNT>2QfOAE9@u{LO>)DgK$FXS%6&?>D?Mn6gOkh^R^ zs}I1Y+dXC&B-a&pFMzMF%&r%BHE?JPfltde}S{`kbK0)r% z5^YkfE&4_3gxuvj&4!dGm?QlfZLTI4Rdkh_#b3AI`Y^~x2RRs;F#LJO2wtL@Qtz1sFcNfjH6Qb;3^ zJ5NULya>6=CX`ZnyOFz`Mrjqhh|)<{k-I!Z?vnK=|3;}0lFO5a+G9L|4Y#ACT{e;|QD{_~EXpmOVp~2EkM-QQ9$FhIv7om#vylEfF>)p98Hl{ zBX`+`+~p9Os@3yox^xS<%L_D9u}H`H`XNO}?vfa}OLjC{t9j8JsU&ij8fdO!_0c@( zQ{*mfkh}Cn^R+qzEs(||cUg!QDz*YGlGY%1*^b=hFj}nD3uuXS8@bC%v{bRkC;0jy z#X#=zE^?O~Xt`GNp%qdo_qNz7VXgLWwcYehuq~g+ND^u-}vex z#YXOu47p2gv`4E2(O#)6a+f-2pJI*CeyJI9mk!8X2BKfJIszS#en9TB6dhD-H992y zjND~6a+l-iuvY&-N2L46UEZLhibX%gPc%w#kh>&D?ve)`*J>ejLMn&cr7k+D*r(_> zsX20&j>ugGp;KBNiB3zCk-IEIXB1n5&Pp4QyX-;kavuGz)vM^7^Z>a_q|;F%omVU- zx+uj%?vfI@OI~zIt3}XdsXTI*`sfeEnxd;xOXMzJA$J*qZfJEhx+P6P?y>^iRcsx) zFKt5ZvJbh-Mf6as*U?|nBjheo&hU2|6^o6YN(qp=yocN+JNj3vMbQhX0&9&Vk`kbZ9ifOFq1ak2isDKYP{fWjet0ho9sWNhx#;B2E zEzqY@Yve9nk-LmQpJ{a*YA(${?y?%SQfwpoLfVSls!=T ztFbSH-@U$v+AEe3b(FFpcP@n7xgv6x`sgd=HAma^9Aa{vzG2A6J`cp6!}`XO9JFBnNcIH=0c67qR3sUqfZs9jhaXekguf`a+hAH zsa6M|&!q2>yUaz+6^>CMj$X!yS z4qDBHI!Z-R#E#Hciq%G)r3NTsM<`-PsEbwypl;IlC}KzGTg8^39@0t_u_F|*Bh*W) zXHjqII*Qm4>Z90e)K7|XBitn+ir5hvsMTC(uv8R9>_=0zdIn9Ct|51MhGr`E3eA!t-wbz2fZQcBnyuAb zXpU4AMeGR8Rjf9eCpAD3J3{W#3(eQ+0JK2*9=Xe0v`DcfXtA^sMLb5xT~45-TKyd@ zmu?_;`4_EJ><#)!ih3*DB@uF$tZ0o^^PqK7apW!^qxFi_ML$c8kgugRa+ls{gH{Kj zjnY`;F7weQ#g?I8q@R$xY(?(!8``YZ^JuGd3%Sb+v|X`CxA}BIijLeRF>;seXt!4L zqCHYc*zU!_lxyR<>>(gPjT>OkaojH8j?>&-%kw7LXs*L%Iq=&)kD&{1hW za_3XXoo^s_d4`TF?=5ndckb{RfMO}oDJczdmz>C5>Z9vgZHjJ4ZIO4`2i;O^FuE;` zK<+XbxyuT4Ppj+DeQ5`Bms9AWVi(Y3=?Zd}2gqHb+~qrKt;R-ArKHGRa-e?{%ZHvz zg^|0INA6Msz0m4s=s&3)a+kj7wPHiiTWKV6mmiV4tVB^0hTi+EN71C6$X!mO7>Zp) zv81cWT^=HLiF%LE*0mZ3#gmdDcgcwoD3%{3l8PXAses(25lXDp<|v8O0l7HurKJ+cT|Pwa(j9%O)jp`HGy=KHOw?Sl z1*nCz6uHX=wS11;r6(Gx)&6L-G#a_foKT)( zi_sWq1#*{PLaTaxK90s}^(-1ET}STn4;ru7YcxTM@+jOTA##^2Xrfkgqe)URUm|zugO+G@Fj^{&L+-Kw zEmLecS}v_d?y?QJ%PF)%s~6Bp={9ngmuQt@kstG~Em92RF7G0D$${2tH6L0hl|t@P z6Kzne0oo`vLGIEPxl2Fvi&lrCEz$(!E{o7M#a5#2(puy$JCM7aLA$hi3GJ5dB6oR( z_9_aAwG8q)pk~VJ9^~~w`IR?1$eWCsD{oGaw-^;r-i9D=Cu*U*LqXn2R7iPOgS`8w zrShHzc`5(lCjph$1o=1X+M&9Nbwiu=^A^KV1I5Oo#?mC@b1)bA9IQdxRj~v4FA^L> z%k>PniDpW#&<5ot{+GL}eksvsQd$&oc9A<*K$BJ51T|A$TeL&59;mru{ZI>OH1gl0 zn1fm>witabtw5gf3-XM^=nJj>hQ5@pAb0s6YOB~w)Lwdv+$G-gaF=w*zd!vP)zlr2 z@q({(y3#agmQ)_q(Q19vQT>{tucVg9oxei9zJaKoDt<&))nx_xS{3V17ikl6?LOq% zbEv**AEK_xdy2YAFOVx@ybM>QM730r5BaySJE4a9E$k`i8`aK47xY*zNB+tA4S{x{ zU$lA{&1e#O?w>&Z$@!H3@r+WJ?C7X=R22EHeT006nxnqT`x5n&I)q}X=!v|`5y*FF z7V;fhgZist6B;0GL#{o9TzdgU+#xhjc~8+`=>>|oL$AUW$&l|*dbC=5%YwYO4#>}_ zUTB%R3`TxNO+l*^n~hdW3y^25LB6V8$j_+b$j_)hke^Xck)KiVUWcDi8ADRP9OxJ6 zedNw1kvrExenxdb+m+W7`584lB*n&}oze{C=fY~VOR`#zEv67ttQAUPpVS zzmU5`eZyxNip4?urG&^`QX_ZCi~Nj=^)`Gqk|95%@}OBd2enaco!HjsfckYr2c<5^ zo%}m<_MoHE0p!{<$hCKo&&z9cLV3|5MUC{E6dSoB8FEDqYr!Gh2s1d)XR0jF?ls-YH)ww?M z@1eCp&GfwNfm%rY&{^e;M*c10dC0$yvlRWV)eXqMk8=oJQ0yeSD4j*#`3>Zqzd_$< zM#?CxN`(T|2-GgnK-5opKWwc>V4$rYK&-6 zBc7`nkiYtsK<}#G2Pm0T8M$*Erd&--H{9K)n(kQkJrIR)yKg$lH42qpV z8KpDGGp-}gc#5)U^*@wViWNQFB^Ao9SO%0+%8J~jAaa-U$gkdake{J%&@A2iOfjNH zJVT43-0D&Wy)RWj?pG7J-{;8B&>qOo(2*#gD#oMy(q!b?`N*{!ke{IkP$A_VM}?)+ z$Q9R6#63WMhQ^2)e&!_!lo9!vmk0SI7DW$r{z@UAzmJg5-w5R28k~!K{thFbzw^lF z?>6%Jdx~o5`Xa{)pTDH2wqmJK9Vr9y)xVE?^&cRgzk0~$uO0IF8;bn3VlMLe+l=a} z-!4>7+K=4%6msWB$mcI+Z0@P@k|3YIOsJ7!IZO8K7NakrE+yPJ`SzAM#18gnSYkqxPz3fjUa9k!!mm*A7KKiPOxO({2cZRujYNy2ACb@6ald5qZX5t&?@z7kA9LmBX{nD+<6@GiCu_%VmG68s@R3rOZ$;)Pa)UdMn19sp`Vo( z$-{O)Ec+Nj!9$R~Cy@`*hZ=p6Eiy@7mUAELZEu}_ds zY?Q>|6I&Je#I`_wjU9!2Vy7aX*hR=Ec0D?#YuSZ-V$Yxxid{k{rEAE$e1yEq$nSz?UBxBO!^KCyezpXzrQJ&;ZycfO3=`7!c|jh-xgVpE}qs>pyINm-F=3nJH6 zMn18hqQ8{a5frHYS{cljyuF54pSvOn@J7b5TS7nEAHJ5d^G zAM)Jakmue)-sMB&UA{$WRS_c<@0z8!DB_w?#5E)DvNTGsyoxA;R28|RK5|7{UZ6~>jr3mBNSUSR$a51T&&`6o%RI=tEQ_+J zqB6=VRY$IEh+Nwqd6#`qHsuXQ*`*Q46_b%GmLl(RBl0e{qa3O@jJ(V9$h&-qyvsPL z!@JCayvyP!w`$9wJW?g(xwVn!eu2Epj>x<0kMgNv7|Jh=My{QTT)P5!mpf1) zrNhV-=a4J@MBe2K&44wOx+B^aOeC zTjaS(GKc48LPM074|#4gG*qz;L2M23+De2SMNpW=;ZwfgNxKE)?cH&t9fKE>D2TID?s@?N5Elou^)I4>?*r@U0i z{W7EO%6mV^D~vWMuVRq*3Hnxf4THR9=ojU63i5iO9?BaSt+9aStjTB&DLF7%sX1<)y}DDvH{i2SJ5MSf&HMQ4=P2KiC#jLs|82f6cjbWyP> z=nrWoa_42powp%(-ixj(?<8{P^XP_Rw~;%)M7I=+oSi%=26E?jkvnHa?wk)L{4Dee zQW*JFuM+Zk8G(G)rlXkehVm97U%^TgTd|F3x$fFw6j!m|P<-iku42VdQmHg@=W58En<95^g_0@nYvgP0iBc#w6uI*>^qyjK zQEF)sa_4o(o%bVmK8n&R?*ek?>nOcqe<62{nv-`>ip4>hrG&_xQzLiIi`=;g%A&jq z$epXB?20u+D|DYbpqz?zMY*IN$eo8Ecbr;s~eM%i`luOpxPCn(~peeS3cf5p5k@-v_T^1Gk` z$nS#2BEJioi%RNq{w-*ZbQo>bwVXq~`rGJVeT`306XpbBCh+s-1`qXmxQ&imgJ2q;)9b4k33whurxpa_76qo&QBg zR2%L6aOV{0sAB2RF)1^0=lsZ>E25XWmfFaVLJM?U6`hdpP;cZr^b_(O+KPOKj-wN* zIEzk77m=skMV|H=c^}d9hW8O4c^|3KZ>r6Ke1}S-Q;JnYr=_aMo$DiaZi~E+Z;|&g z6rEAUMC5(UMc&70HE1X#?`KJt*SQN4{%UQN*KG%Zi?KwGx8nki+qQ^M~_u84f&B?f_#TABHy9A$am;J^tURa6pk9{i4+rgS`y@G zIgsyAe&l@=N8U#j^gq?sL*7SQ^i;9V=$Z5la_51_oqt5`JR7<5V&u+SLowAJMBDX> zc@h1q*md+=x`W*LDRSqSMZ$L|De@i4j9#cBKk^+ai+qQ=Am5>W$aiQ0dZ~)3p_nua zdD?R1X*-ekaR7NACy@7X1-(-31LQjtsc6(luN8}l-bnF~JEuhMoE!O(E`hv{s_3mM z>Lc%?74klQMBc{&rt@XGSMU9nL1~ri?AXn5xp7wPpri#&lrXv5e z>nhY-wOi13-M^#gbHz@h*3x<8&bN^}zd-G@8l!Z$HU;XaSUU8Tlo`1;KXPqFG(lI@ z2>DF*MaO)l$aj1y@|Dg-etb70-}@8D_x>jOM$`U8-%5{>=e|as8>b90ttLml_qos@ z#R{MyQqfSJ?nXuA+Gfc2zAGA~yk2OG)DO90G;+ntP)s{I5a>O5Y*Z4o9w?j(o?LB0n-4 zkgs$*^1Z)=eD9wi-}@Nlc+FAg_-KKY7MjI6S z7;TbjBTs9FJgp1btkr(Vwd2uN#ioQ}(oE#qWyrPLknjCrV>4cFCQMXU97;TH1O zg=ffL7t)sxe>Et8UTH=t{QabPq*M8v6S77{!pDAzyQp z3gK%`iQ;HAEAq5@D86DXgV=23Y3q=u?Fw`Wy`zd7D2a3zdD=7NI~1=XU*EKv5_wt` zltQt3L2Lr@v_;6%)(1L{(x~DrN+(@Jo^}^`+FO)CtMMv@ro3MxHh;&_+~1728oEX%F(W z6Ufu1s@>U0Vo6$h!?G5q{qi>aW zA;`Ol1}X1RkQciuKmDYtuJK0|(g)d`K#YIo%KSHqD%v6+JWO1J`zRo*%@UfP5r z?hEpL`3+6f>J{Wy!vCSkioHZrrMJkn@v4PuQ=%DK&5Byj#L@Bwhr># zZlRbe#s*pzXe%15`*H~RUFjJ#ToqSPre>j6hMUN*Lr;+(-{RH7ckL78yY?AcqHAf5 zeAjv)-?jc|saC&7zJGJkGR2ml<v0D&%{R1^FK2MHO@p3L)Qvipck14Dvl#gzoA&y$;>fv@PhRu4*4zsnz4? zyjCwE|0K~Jp~oNhQ!msR24CbwKB(EoiHBDA0MdMX|eq{zd-R z;W#zJN`@jQ3!U~%D7usr#gy`*e(F*R`JUE7zbn=p#Z_KMbV#v5!RjdFz5R&dD{lcx zBrQkpN~@7)Y(t)L9(l%HlvH`okY^;Q6<$q-JR>7Ysl2@CJ*fyvBb7v+Q3ZKME94nn zQCj8oN1ib?Se=VJVJM&-q=9iEXHc{MlkjAAI0 z@+zS$(nlzpR10~=XUH?YL1%pL(KKlc^4GE{sIE@&OypC%68RJ#LH;+*8)%4L6Jyp1 zf3?bm{MD)`8mKF%jQrKACi-3#jZq)XZGi?UuXB*s3yo3U;2>`l8mzo&LEZv1R(Y#} zyr0n!w=~!uXm6)2#r_XgdlGQny$P>LEcIn-vQ@*6<1 zCXv!0*WhV1PPxyb52V7#D=Lk=qULC#TDzdty6#XkS+P-Qsx%fkcRF(JQsioUh9;^S zn~QNTB7w%MPBa`-ezWeu) z*PFFDzY){Usf)bc7RYz2H|n1uwqpmPfznXq8H_`o!93(|{nn!)>bC=J(S7>~y`$K1 zG(tLsJjzw%QSKwJBUuZ+>(e?OLS9D}_<~|wPVQlC|S$s z4yH%b)S45ywKMV^9DsZWM+HqluFE1cL%A!^ENKn$inb!J=r=S+t$!fz;)Jc(O^PK$ zA4?A*=VnCC&5K-(BIsLHqd0OkDj`>6G;%d&BUfV!ay4!tSK}V?E>7M$s=*UzrDl)| zxdvU3Yw#9w4MrfZd>UG-jJar?v=DjrYmjHZ7kL-|hBm3+d9+!&f}C+1IU`k@Xb)va zThy;WOxi=QpzVrPKs%)>$fGns9;E|v)q0{6s#-tfs*OUf+PBD6JAqua`zU$A*lsV{ zHmX_~6;T3*;GeMV`SZL@Ki&RB<>u@iaD2hjgC=R?SI zK7%~xJnf=6mqMO%Q{*{MM!wSvkbjO}j{51kn^0$cC%pyv4BLnLtMw@I-G8b*J6q>? zA>_MX0=Wj&&;VuBK?9|R$SZ1xyrQAVKkt8p2CLr;G*p_4oUsBq<1q41q-W6x^}B)m z%skkEpNbTF2)!qzK_2BvoOj>F0+v9vKYB8pQCxo*pB8)yOCFU2)Qojkw0hd zp#|!fyi0UOWI_uSdkQU*o@}*J=}Tt@b0=>MF{n z)8gr_QLUasUQszzCzEy^sxQ?*o=a2YxpYTf(YvUz`b|VzboUpc=87#xt)$h+qijJQ zWk2#denDllj$e`2@dt{ZAa6$ND23uD2=Y26qxcDe;wK0is_Sk-@e>5aPY^U*tw&M( z1nCyt=|U)ef}r>bf<`N&4tif|h`ge9$SWF(;wK3DK>cQ*3DR8Tj1|Zkhf(|lK_98# z4HQ2?y7TU+*h6Tllm>Z}Cy_@fjN&Hw`~*QCYc54m`~*Sq69mOi5EMT_&|+n*MxRLQk=L;U#jAn5uP&oy>h}-YqJ5R3 zCqMBi_9*&PN{2j34&+gaB3Gjfay2TWcB;l3$kk|$T#ZkVtFZ;S8Yhsek*-%%jhx8U z$dA6$42mP~u=2>KN@e6)HAMT>uN`u&Rw38wOXOPZMy}N{^n)@^ql40UHOyvn4# zqgrJ~KdWCJ6)llSGeTZDEBgnP7id?JI zefaI6PL=Y=Ph$<_6}3fgXO5jJUC z$SZ1&yrQ9Ko?6Et|GBh}(L%*OL5ro8$hn^*=N?5z^fS&`Vym)hLHtgNA5@GFqUOQd{J8 z^h93AJIJ#ii`HxQ6Od>BG4fr#h&=oI$g|J-)&mKuXkXPuo_!1C8T3X^XNm2rf#?}& zDDn)(AxCzMyQ<0!v$3VD>P$fMjxUPrQl+)J(FA>?&rL0(4# z)L=+#R$Yj6}bjg&=u97267Erpm+^X z1J?k>Yk*p6kEa_N)gU|aI*OuMy6($pw)86U45}l~pc(QEI-vWSL09A%3__m4x5zU% zfjooz$TKK9ESf=CN1OxrMyP6AzC*B_Bqva|V>UMeI4xid^Rc$aU_I8t6`sL$32m!*MXqUQk)fx+1TmAF8f(yp4R%$D;Td zfxM0rD1JsDucPRD(K^ba_!)sJ=(-J1{ER^HGXhmq>i`r#BT)Q|K=CsI#m@*-SsAxb zRp~DBijut_t>`HfKO@i^>Q@rgkjfxuR70*oUlczhP%ZVFh~j4is-xISR8LxqJjz$d zqg+JsGXllWh%wQAc^Jjd2-Hk<&Whq^1d5*#$aNlv;%5YMozJ288G+(w1d5*#WBGGI znqN8;KO<25j6m@-0=cH|p;pTH5VeseBd={f^4iuTKRw@}cIx*F+M=iD5^As5b<{z+ zg*?gwz+pOZ+4JtRRVQazY55;8joD7S;)0oj9jbFQ4eKoM?Iz8$g4brT&wfQwYrCTsbBIB zqFQA_y%l>3^^u-I9;FEKC>4-vRTH^ZjnVt6RSV==bw{q%CgfV}N3PXXG)^bY(;r5) zdJcI-<DYTcV-GYsMZCc^$PNP-5umRlWbz7G{`G@1|3#zfmlp>9(hHtAg`z$I;PfM zXsdSdyXd51W6>#T0&?ygoODhE%y`TYHUPTHJ6>pwR&h$RE-sOg1*-5 zdmzt#81n4DMZT*ikY|4%EzrIyIysts8RQw%K~)mQKEWEJYEld28FWRS!BFHUWion0 z{pO=Bx*zLMO~t-IwWMvxqwGf>sXdTs%*U=Su9gC3f z!b;RkSNj~f;unzb$KR;AS`$x=TB{)M&W6bMqkT{hsk|3l8rIxX6rd64V!JUZJVR+nPPb*YG4m(eJG#v<2c3vyj< zA=l*|^6pGNJ*vhNsIBIb3%OQZkgM?)ay3RE*I*iIr;NF%y|fT{9cz%+u@iZ397P?} z?>yR~y>T0LR4l=a2NHCWk|2+g26>dHkZX`1-Dwf4!E?wpD1+iPK=B%&cn#2I?eSa4 zHApx!T1Pr`Ti4Bk?n+M~&!8ak3`!%eGB$&1$TMh)JcCb=XRrl%1}BhbkZx8q zgPh2p`}tAIM6vOTBY)DBLf)rUQ7W}IM7}d0BY(bqihO4_Bj1?==wW3XMvq9xkXLj8 zc}0n4NAFjUqsP=QGfFFEN6sjKoKX+?&8Ra{f3o3U1%%KxIwEn{^_G$*Qa8ss`RK(2FF;gK5b7WfAh5(K6&!ZbmPv-)`hpKKyaC$}GsMdXGevcuqC>PqKSbp@m^c?airI1IdhP;mYs7cG%I+`G_qYH|k5h#8}p!gYqypD7W zqjls&@iPJ)*L91d_!)uXX9W6HtqoE9j6m@-0>#e=6h9*n|NSHLJB;{G2GKF(6N9+7TsDMt77{>Vv$tcaYaM8F{ZQLn+j6Bif>;XAerL*bnF- z=`ivrzax)w3%MEzKZ&Z561}5pJc7LIvLRQa19CM6Ay;EIay1SkSK|zFHLjqiG=tm7 zyY9iI(MgjMxmFobPW5{RxmIr?@4vT^Yc&$NRx?m;WqgeCN{f+K`5E#mcO%#8I4Yoi z7m;gq7Zp}4(K7zNxRexml(fjBJcC@T!pJ+PB$}pLl|imm4dhx4M6S?8v_X4q1M=To z`w6{1IJP(LqA%6@_;Mjeehd9Bq4VnZXXtkqwO79s zE2Ht!q6_Mm6?wc|sH6H73H?f;i|SV~^sA0Kt6$U5uRXe?emz3J{-~?^jY3|>1aw*b zW+GpAK6+FA)`WgvpeyRPEA-okx~bps(C-}jL;e02`u&Z%tKWmGqU%0_uBu-~6#w(K zsG0ipM}DH-NAD;$3Hb?_g+?j17=4sKwqHI+?<=+)jg@vI@7P1JnC{ydq#5HmwUh&bh$Rn3P9=R&=$h8r#-}GyNJaPxbYZtLTXp-jq z9^wUp*oTM@N;DaH-@pZy|J8tuG`0UA!9Th+_58aj6M%Zb#(YVW_RHHY?UDZ4BCl z{5SA^K%>-p0(tg-BF{eg+GzHfk!N2Bozit*M88WVk!Mf|c?NaSIkmPzo_$|*NwGob zvh)se?gz-ZD`PRuYFE%9Q+NB-AMsw3aG7ARlh*!}H- z3P}CYbJ7U(yz~)rH5QH-|H>g}tE#wuoKweQNbVko=H{@q^2=d9g9Qj#2fc&hULI*X2%gCqvP4tsu z2{uQuWawwbvLOG9K>5&a)vp9vuFt$GXr$ zbV?cP(P`;(_dMl;}H5wI)a>g4mtNW z@?6q?5k0>-k@sm4^tW(9`}Av+ zAW3Y!-=T!k_sF@wBIo{t{H&(>@__`2)$cKsM0x@_BNuW;`B+SKX%RFqXiCsRjdPgEkaKywgP36)*$C@Mb15nT(zs{N%i{+Jtf^i&Pcj7%E*mewO4`~ z1@#FUj$E|~$W@zzymMBe9JY^*O3|JS8E>R z8oZ32Q>-j1ELB2YQC;NRzOk6@<@lh*L0g0NA+P8d@`^5^3(EaFDABekmI|fn7~8v# zBA(rn>x&nkn}Z zYA#(v9_1eLC=Y)fRV_kb`{fNBR&Y^b7_ycv6ZXnO}KjfLF+R2-TS~DPjT9rqwG=rhYJ7PNOp?>pFFKH3- zcx#cz+k-kO_eWGoI)nNv<1!i`{SP_!U*z0Jc2QGhWJSJie&pR*9Qj|wsDS)Q*9!TQ zsV^F=kq4on(mTl4_yGAD^U)SvV-52E(6cqf4k6$Diy@YJH-Ar98M)EB8o3}EEfqn& zMj7O5G)Mk7Il3ahC%%QoC}R}zw|>*nc*W+S3DP1IuPO3)yU--H9z@PPi>4`d1{qn33(q3M=RCueY9Ho0C~Jw$m4y6 zy#KyH6?NbCqjk#o39XloBIlk*&b@=Y|B~&AuA2^d|7Am^I>yewT*&9&3&`hRf82Rb1gKqsY&{7CNF*A41~ep72dpg6NELi=wmAOUR>CL>{FP@}21r^cM0Cdmo)s?nLAr zHWzu7E0K5D4&)uS2c1{O&uE2q*p*mJO1LlDVUHs3uw3YZGV-HK(sRf&ErmSO%IFWZ z)<-_!Mj-F7uaMVq1YJ|V-_UjGEb@3akjG2DKiVbfk#ln)?~=mEr_?LRr&I&vQ>qL4 zQzQ36x1|2a*LWBC8q<(ZsU^s#)VdJcgM0^1hFIbQ(J7S`{iBg{p*vDur_a!Gp1VKLsU{W+IRG3G#SfAfGh5P!ByR-yuIK$B>_t4Btmj zNz{dgVuevC#w zxn`j1y2euE^?rrC-XD?Idm4GYw@__m{EO;Hi4I2ddl-3snNdTv=0Wi{TGT|bvZ%RK z33+XGk#jr5T6K>1LSE5ORb%7|^+2xBaO4Vogj}H|XqQG=g}#wKLtfD~ zLN&ay71@?-jd+ew6Ma=O#NGcv zk!R2idA|%p-Y=ukWo1l8uFz6+O|jMJhO{1eydB8n9Y(Iub>s@&M}I3LrxY?R;&SfRBDQx+X*>$5OQ6nA=hOQdR!T6k$28c z^n_x2QAX(piD2VpX(TvD)ZcsR8me+9F?L7@DB@jYU4Crl3rnVkgZ^nmx!XI)w6RMMscVbP;()c}_(uDuukFYUrGidGxPh zCD4CTY2;C=Adk`vxh@@%>(UFoq`LG+uFGiTx_pa#r%xc)(Zp$g?VkJgb^0w_2Ma*QF9zA6ul&!L>}b|idPr8E-B7LbxDgVt1cOk>yih#E*+5T@)mkSBacA-S$87x&)xHa zRt5ct{?ZA21l3mVIrN?GQRE(F~QN_akMO^KRlzdVAxU$Pzgg%r>0{)KRmd6N$6~7A<)8;IMzP1ydgVTa#;ac; zN28*Bya;-3e%uTIV3IaxL1d*k<&(v=upbFLLgsSWI(C zbSYAnpnS;lD~CM4M#%Gf6MdnP`=BqSw~((f3i%q7&=$3Rj6A>fXscphqOYXw$hilQ zbN`6NG`}R5BV`LJh_)%WByvq(N1k&tv_`wBC)%!l1JDj>F!I&LAYW|`@*Dh$ppD3% zAKOviPO(pe-N^g?XXJf<4|(4|c_rHSMUeM>S#(0zt%AJoYoXs1>w>)RhXhSTuK0Z9 zif=&Ym9Yg~l6D}k<9pm}rh-$yqTOY#Tb6G|zNbJHW|7DnFp<us>e)xaU zzR!W4Q@^~Zuv8HFY9*1cRvmfYHwkKwyzhIUDcbjak@x*b6n}3*@%JVae{Vu{b+uc_ zyZv8OU$Ioz$&fOk#!_C?Oe&6=OBGQ|sUB)AwMFfucTf*iXe#oz3oAlwdx-rUVpl>e z;f*La9dd4N6n}$9e%`8}PP$qx)K#jFT)#HR_3MGUsdX^&ot}t#C^jASlI9@iE=SJY z9c$G(jt5;reoMQDJgZbUqgiD`eKp=QsGpP{dF0~ABbP@5)LIL9R_)N+igiVUq@FS9 zoE(CjI|F%Ep9XC~p4GR=d*dMbOnc)OD(C;^tR{bu8e%H~L>X+bFwB8ixarH}!yxvS` ztNP^){feM;>Q^fCD~G;PzuKW+Q0FDq2DyLL;aS7err%h z_4^|9+ktkf-;bf+ag;&*&V_z|pxx?sFZ6rxZ@v>$zekYQn;w0meorIszd|UJ`jrU% z%Ah^!_eSW~5M@@s)}db~v`_s8gnq+O7WEq!`b|Oy)NeuPw-RMlzm1{a5wu+Wa{m*Z zwzbeP#Tp@>vHg)xi-{t)Jk6gc&D5qxH4tb`1k!QLId8Yf3XL=45(0#j!y#MZ^LW(6z6veV4 z@4x4R>L9Q3UF22HMBaa^gKiVK6x0W~enU`E)o%oH{U#yT?+9}J5+;f2R}gvSeUMi5qLBH!~G z$oFj+^6d8_&;D9anq*O|SWwfT;XzA-enp=B736#LA9_i%dN6t71SO@E$oJ?8r-+EM2`W!jq8{~{*$SXRByrP@PGtHASnqNcYyD$`aejgxT zZ9b~2Yb-_erB%pR`vUoDKOkqEL5{vsZXp zu6SeQZxRM0*ZxB^Ue}m~K9oK|6Qy-%va}6NmG+@&(h)RWI*VpW*O1Slv=8w&##Het z=z`P+%~ik7Xui}Pd2NG`*ESt(Q^rcPNd4BKCDLZ(jNQl?Cy_I5qNVD02Q8QWL(WK* zI?Bk2oKXy|Qoqt@wNw^4qXzOdS|i{6UTBT_4McuNnufe%Hz2<)?M8k_I)>IO<22eJ zokzaLpUBrp_;BI`o79>bxso}N|J+wG^o3eWqb*Wd6rU-I&lGi1{W_yc(jfGeGDe{7 z(tF6cQ;>6)BHyiz$nQwIkayTG$W^<7c50NHXt(qa@_0!eiN?!@T$fVFb*YA2mzHRc zayy{!q^`)LyoEfgQE0DPCn48m8FF2=q62E(g?^B}LmuT9gZ+`8!I{X<;8x^k@DTF8I)koi z23L^x)j#N(VoB0OD}MsHrbUr!S{}Kk4bcr{v_Lndw#X~$iM*n5$o2ae-BQ0#(BINZ zTN`Y7v!qFja;=4QC9Vvik_5aAaz)&%J}H}b`1kQNnHEE9TB%7&c#EOKs1R9vlP zkaO#xl8QA(FH0?ubGstvzJ*Gwbrf<eQH=)<0Ey$yMhdj!aSS(*`&Pkt$ zlokzDEDtKL+@k2Kys>^2Q6=hp+(RO?{m?++%T zT8d3awWT@8xyzAr_r+rQV%4}1bPIWv2{T5kOo>itm5(5=G8^(LJ0P!e5b`R=p?bRR zWaL#YL0;uXTb@+yZS-^=&WRju*^eMpUp zu^D;gdyrRt1^KozZ^jE%bx*9y%mV zLqAJP&=F|^Ixg)(C#0XyZ_*ibTDpnONQtr~PHA^mgPseHh(Wzh6)Vy*8XpYpTO zgUVQll1WREb2lL8{t}BR_h!&PsHtKJo{HW}lA%h9WkjhpUUu||^fdCw&m)h#0c};r zcPOp;{fN>_KO<+HLC)xyBU;BGlu7+Ype)jR$Qe_RGnOOozs<<|Z!dC9FCo|T4suOX z=j30Y(s+-f98yN)k)J^x`Bmgf){L(SxkuM;R+&NcN$swqq`i(?|q%p`D(~vV(BG=^$F5cqhWvy!MUU$pxGnM%+6ARoYyk2TIt*n{YvSFEe-lU=vGkX+)>7>L9LL#dFqZD>l%ZR>oPWIDst66 zMrBl?#mE)ffLx*5$me+KJW++Rq7u3@Pa{{T1S+Xm1>_1 z!%$^qOo~a@or$VQ^N?q-5_tyOknjEhtac&S^eAel*m>k?Ttn>@yBA`K z3-a!$*rUk(oPNi=MJ9Xn?Mk6TKznLB4KLlrjOXbkd>n2g42N6bLp z5zCNw#6{#CaUXd{q%KUZ=8_J1N90DnZ$**!czNVI*bt3TMhi4vYKy#%p2+JMf<92| zSmZl6ANdY$K$FzE1x=B5Adm7r@+iL}&)_EV9ZdQ>-*GD=HJT|shMb!XIk!wKrn}W3 zsC&@ppee`|Ux-}sPmyc787)^$w<6c{0CG+5A^%;T$BOXXjn4KLk^j_7Bji8z(hg03 zB=+9f2l-DO3_;bE@gDM@PWTYbRKK~Q-%|93`h6DqeU4_S-=5I#5UQbmCqlooXtw(O z8T#EvHPtWK3(<8SMsw6JEAsquqgv`$IP@!y=Bi)C(61({t$vL|zgB3T`t=C?2BJFZ z_ipGn7JaOKGef^csIK~b8v1QO^VM%x==VLUr+!C6zf)*|`u#8TyMyYhU*e+CdQ+l> z>X#9Dy*W_>^(zqiy?_>}U)j*F8fvJ1^+UgAXtDZr4gLC|M(Q^#^c#&nQNO97-^Zx2 z`YjLr)}m$Vw>|XRi<+q4;n43_v_k!^gnqYBQ}s*mVzl0*XqEb&=YXsNd7b&s!xlN&Pw_|Fk_G`J7yU ze%A@I3hkD5pp(kjgMO0^AfG9a=(5xeomIx$$fxc5$g`h^ZmQpU^gro4^oR5ddZAG4Y&wVX zNLSH0jdvIMb0I|uezsRE4RS0yx}aE|5G#f*D)vffglBl=6NeM9S*&^itI4$ecj)Ndv7-TEEv&P4}`RDpe>}mwu>#Gz9&lQN|*#@*{Lxv3Vi39=)QBuTeqi2UK1F>Qie*Q4G+qJZ6_rHZe|1n zp8CCl?n~v6pPrh?PfrK5QCI7WX6YJZk^kk4c__i7u?j6g4@fJJN7;-#%04thxo6O9 z>2`=^E6s0=b&V1znMNstQb^^I$E$@rUPt7A$*wPYSp9}0&uS`4qu3nuxU>Lyl-02gYl!Vbe)Ihml~n6lYtyL~>2|Ez|*+wY<#T2Yen(QbbTHB&5Oh~-A!*~L%`wZ4M9v+E%L zOxFapQftRpKh1OyYNOZ))K+>AxqefS>$eKIYF`KKNB*}ke-HW#t<}i)PzQ~dyh60^ zv!ITOLztS&gg@jF&a5zF6ya%i%~CWIdaA(uEQ@@9Ya-v@CPD4cZ@R0Uk?-n2wx~yY1;?+4R17hQRnkyJ0g*J&pX- z=BXY%wWW~H*lK8_=3E!~ZnZ~#d+3LJvW`KU)Ndl%EKNl|hZZ8ALu=6IYTb&)scI+D z7mA%jUrLvebN@lkP5B1DLD25ZhqkC+VYF3x5jmqQaz;J$q^{8!ZBxIVXou7fIb#HJ z#su`WTIV3oZ!6lV*eP3N zv`4CsoY4k3qaX6;?>orn^H}tP?&<{OyZSMTf5M{pCoEe3c2s1^M*krf?KZZ7nrloxrF7m-JK9c|G*Xo~#rBlSef^dyf*9(gJH zLj882pETYcbXYooJo0hmk*}ej)p`&4-~UTpn-fH_$I($KBXaID$hjrZakW-J&TW8B zDApXElG-5W_CU@ZfzGOR0&?yGbV0FY=(6-Fa_*PNx!C_ZqsP*e&#@bQd`{ zS)C~N33N-XPojUM{AiOZ`8xVnu^K2ry4YT;i#$pzm zLRubLzef)%_A^Q&9Y?;#MdWLws>^Sn)%qBELV5zZUoPZ+#gPA=aCzjP5o;sA1+_$e z%DNz*wtZ2>-m$d}M1J1JAU|*4B0q1((2Kh6S>!jFo5;`G1NHbxPOT}CpSNtt&s!nn z=dCO%rGAxBY3U8*bu>XM8aYsxRF^&Q01N%1w_Nsx>QWEagMqb(K&P#ojx?v_zgk z57b<({ZUKlUF1=gqBe@HMs20_$fN8)9_1ivr`DsWy>uRVl#~tmy_#ZaP-iJU@+dix zM=653s`Vw*O{$1IN@vtdv7V^6)DL--5y+!VMScPnAwL11AwL1zk)MEl$WOq}sI{Je z~T~`v|!P zGtesaTN3*H6#D%h`dvb6)$i}nFK3fzyu!%i6-VpTuRQW~dxm~PL%&gIqxww>{kDgG z--mubqtDgvOz3w5`EB@_rqP@EP~>;p578NY`prRp16_gqR=NRwrSZ0)9nyEmd+k@W z+gC$-q>IRV?H2N0OWur6Y_+CC{%t{SbWpK^=&)1-IkyaQZf)dRbwGZ*?vFg@;pm8R z-$%!!sVM$jqLYfPMyI6p$k*6`e2v5Cv|4{hp5I^Scg60ZGg89lQEqDF-0aBnD;V@L z@_Tz_?*tVGTY+Xe@8SWvU49QzYC&QztfO4}UuPs0F$?z(Qe=mT1GPFkicA+owPpM<}s?9YM}Lhn#y0y`t8C(W_F5Hqi_+qq2(SKoz9i$fLY~JW56M zx?1ZX&!8Qus#sU_hSU=|cL;Ls2dIWxr=XhB0^}O3N3|6D64jQrBaddH zGm)>c95q$zTGU+n3VD+pw`j}Lfjj zJW2`VQC>$~)mj6+DK$kNr5oz5SYOmr8i+i~Xyj3*q26kpi~2|_kVn~y`YE;x^_RXw z9_1J0Q7)i?YP}X~mF^*rlDa*A`dzWd(Qqjv@+i+BkMbfKsn$|xv{V^+lqP75Vy)3w zsRQyTePS`4FYlrc)H)VTkftM#vK)P+*jhAM+K4>LPUKO3LQ~Xw98HxjB9C$xO;apU z2j1bNq{ySBMII$5nxWQwXr@#Qd6d`DEX8V|*-~BPQCcC7(g)2|>pRHr-jmSBip@j| zqE0J@zAwLxdkUtTBMgBy*jQp+D?a(i2$0*}*GR5$_{^BBtydeIjN> z{zQBp{h_P9g8YeC4f%7hE;_7hv_k$I?1+9=tZ#^oLH-<^hy1R*K4=H>r`kWrpK3L` zM4xIckx$2N$e(J1kU!N%p!o6#s4&_%_vp-WOO+Ud~Uppis>2?kz4`7-dJy>i#D6Y?nMQCr2XqW02FJ< z8lcwI=xu2W@+jY;wsE98@~JMu|5IP@DEGz0l0`~>+V+z?uK1|1AK5p)4{)JgaU@=5qF@<~{{Uv&Oe zL;fzjDO#>|v_(EG`XT?8aTM}*;WN<+^_!1YN=uMez7Bci+tDhu9zZ?xoz`LWsbar} z*d?@DvA>bODNos-KWnL28njkQk9>`s$k%ug`A_*&kHvJ2#>jul=S{RuxqZ-j=`G|@ zMj?-~4AoHXSE#0xb^zb7NO{m*ov|g+W{vV1`dq4rJYF5-@j9aal+gp-k_MnBHQso% zMY)sFR%r(EC`*t>`4V}Qoyeo?L)~;z-$3=AjeTwdE^N`N=zt{N&{s6g_z_ zq3gOvITU~MKz!HP{lKCk8?*JT;|YJW9HuQH63K{}!(R^6%QpBL5bzK6+N8 zG(!cY)~J{ET6fe_8jk$4MXE@_wm}ykDjvzsoN}jWoY?$opjn@_spp8msjf@_zXTdA}qZ z9_^RRsG0iZh{dGb$m@LpdA(JU_e*WmO8r_O@0Z@FtzrXFJ83BLDC3YvS%myVtwr81 zTafSNA>{pX0d>+S*H9PfPvkq8a71(m(<1Mer;zu{bIALp6!LzlgM5vSA@(8iZ-SSh zp1Q^w)LYtse6_D*t=gT(P(QU^K;AER(A$b7e3yGJJ&3&W$BGy{1LEeicUMVGaQ)*FQ?M)5NT#m^iRKXcFq^_z$`N>h>7u@HG3U!wS#gT7F|A5i?vL0>6$1#OeA zBad<)d6aacqCJ!Y#m^iRKXXw0%t7C1lxAp;)EdRl927ruQ2fk6@iPa-&m0s#b5Q)u z39*Euqdk-f?bS7&Li?m=Q2fk6zFI}JU#->A_flizcl564N5y)hL(%}`8jM7)!4&kf zT4$pp(lX>xzC_0q`x^Z!eTzKGVdPQHp_6L;1D%rYAdiyjJ>C=)dkp<9J%Kz*F62>O zKxfoi5}lPQA&=4sol~qOIxn?D9;FxZD8tbuwZ4xoOH+|YS&FVIwi;cN)+3Ly19_B# z=!RO4qMOorYB?kiRu{U_B%9;F5H zDBV$_jIpl+sn?+`GuR zsmCNv@Q_+FA?H4eQY%&jJt7rH&Ml9e+W`5@X@`8~^hQ2&-a$Td#)W=UQPRG#_0B>* zb5@pQNPv5XU^}x93%t?)W9^^zmbBdt?8l^NU zD3wJ%1!^Fl0_jn?vjr^5@oFR7lrIG@kG4q@>7KON)H9 z+~|3=7DO*dC6RYxHT0rlbx?7sA#x4cA=jWUDxuav=q2fWSa z`}r;8{X80ZKYtYZ%|L_*y)s>DQk8%!ql>5m0 zIl}~6^&VR!s0tdaSVQCrwMVW{PvlCzjb2uTh9OsI0&<0pAXn%zYN)yVg$Q4TXQBo*Fyep8X_w;Xj(^fK~HUq_y4Jv36St&vyR552G0U^GS= zj-2};a_#~&RQq64ET(?D&=&o?@CzEJ*hw^AI)gmQb>vYJPmL<{Fmi=5pnj@LR^+-A zK(0%FzQ&X_bhzDz{Ehx1dqTyLdA4cV7#T@6jsc zUAzgsr@OTU`EKn;zFQe)M0cwI@|`J#zSG~iD35$+>LWk3?UC=yK(tr=-a-4MQOGNr zguJ4U(E+utLcSk6(2t7kK?kJ+$hpUnb1$MpYQ2Tp>KoU8(P70>%#31>qF)rt8e+N7 z5yc9JSaEbzvGO5S6&+WsA!?^qmzma3$fAYlwy-ZY!*7L*wPT&hYonkrTnHIphKU$-vubvvLIYVCuz=o+I?E5*j6HquAP zx$}^7KSLeVx(zw^AnKynQS_#C5;^w@a_)bqyINDtjdHW1K8oc+{iM9exi6x4tx$ip z)bC9cCf)*>*6MZ6Ghg{_jRvUdU)km(<6uHWI$W<;wKPvAA(lX>KKOtAyiCm@F3%oz6ymH7@>YyTuH9$qBM#xoKAXm8vxym)@K;_+r zTxBRKrr2;)TpEd7F9O-ZVhM6OZ;xys3?z4DqP zSLuw-RjfPeAYF<)mg|wL3_-3k0(Dg0c;qTGQ76Ubq4T8okgI%(TxC6Sm2IfA@`_AN zs+31v6swFbl&T|FIRUv!E95F&(M8I;0(mUAqg=)MqaM;Atc9Ni?Hja=nCxFjT;*2eDi0x78G{BXZ!&U~x6ok47N8;0N61xHAXoVnxynA%c$!dN zapWpTqx%&*7Cj&xhg{_pum|V)vm3r3aC#j6$w59r=5G@1ZMn z-~Ji$dm}%gVe0!UnyWj;qObDQq1d6wv18C9iXDd@l}x$kH+My)R(_nm{rD)ufKFMWXA_Y35{>(C^%{)ybT=xcn7P_Ytdic|)< zZ)N1Z4baPKZHC;p1A0}lu4uZ{9l38Wujzf2F39hzT!s9;%1g-aPrifv zzRF7E_f}0!sJ&us zLaaABPq8~gYz*q6*yIrV30^pJXq58)LauVqTxM6XL(ntQ;mB30B3EgGT%{Enue{F4ReGW4 z6uS{klKLW78HilvDdZ|IqL-966S>MqXo_M>(=ll|a+TG{Rdyj)IrvS!>#V%;$W@L* zuPJsSnl3d)u5uP~mF~z@u0=DIcRO;G2hnWB9z%1aCy}d6M6NOqxyoYnrt(%ISJ{l- zQtU7EwzLDe$^r9|DwUC|)I;wn?^NU}?a_S2I->ckk;l>>xynQ6Bjt@ouJQ_6 zqS#FIi8L3v%7@5Ren77BD_W|&-N;o+zr}m{ij_xSN)?f-)JCq-Jl(3RSqHR2dAZ0{ zZbB;+yB&QkfTgY+G8m5s<%ioBgvITWo{-qFZa zPC{!GI~A>!PDieC4sw+%kjHW}TCcnT$WVIy^mbwGvq28kgNQS{#M@p?8f6`te#Jfpxk>}%Dy@;LT!8jf z-j&E>xdR=b*Z@>a8jM_JIC7QA$W`7zC6xC8a+R-9X~lj-hf3>^tNe*vrTBYEl?tew z@@gViX^M_i>`Zi&)DpSM`N&nSLmtar=xF8Lk6h(xR9&$N=osk*R zyq}S)WWCQ_fnxik`cg6ED&>%?)JLw8jZRQr8{{e%qlSuIiB6HOL9TKea+QaXt2~2F zRo+X;Ro+I~ihY2ZNgpFu`3kwpZ^%{lozJh?lve_|N)>d5Vl`27sV;JrCdgGfB3HQt zovFOu$W`t^Efl*Soh1!JuJROemDiA~yo+*__X%>9pU~Ng{ft^lTac^lLauVyf}~0{ z)Jl2BBUfpGS}WEDwUy3AuF@U3$}Pxaxfiup-Xq9WoW}(3Dj9CgIuLDa+OBNRdUb;%Ikz&t?4qV86qVf($u2KtKt=RFXmvl06mFCD*E<~c(6nh<|q}j+-7Nlc(b^Z$Z z32j}_@5s+;yOE#O4p_wNzMj>JBR{Jhh5W468u{N@xddIQd-Or*c5UMYbel9Ct6eV;=Y^EsmS=xF3vPxOvHOYMvNPRzYQkD%{#w4OtUtKLhftTYXI1oMzb z@DcJhRv>Rt64 zIv)9bu59G*oU}rN)uR)t&^Nt?x+0%%J(171Nyw{iF7m4T7}eKUzCd1ezaX!=oye=M z)Mva`ue`G8M5zMu=xZX6{$w;sv(X&gpi#9#FDTYI#JZ!&iuDSy8Y=u2;P~(T!4n@VaswBMH;8C};7LpQ5( z1$2vaH1am;B5$J!@>)9^x$gz2uX^-Aw@Oza_q_?Z@7>6AFcf)SrXio*8TyHuWpel3Gz0y0#^_C#l`wsb>*@#@P^zx)$ zTjY9Ipp>dyiyoA2M6Q>YZq?k6PseoT%?nzFd@ig;zWV%*d{*p6gVp1J6@2EUv!Xc4 zf3-(GE3QI5EAB!*D~6)U8p~tIXT@aXvtkbN)#nrRit?7BSEa9!N539<^gEHyiv7M~ zH7TzY@>x+0%~q^7nk&^uuF@2_N_*t9q6?a*yvvbicR2Ej7#m{ikzez+BF}h{m5E9t zzvdl--c!Bf(EHK}$Ro%`9zh%AZFEK6Za?HzG7EVdpP_}SvI2c1eT!W0XOurL$gA$a zuakOpk?ZwDt~UUEsw(%P&!q>E>y1LLHyL@=%|agEx5)L%f0NWZ0ez(^jnUWAX~^|j zBiFk=9n%_mAZQ%&s(S@_)y+d*bsr+Hy3f&#T6HUsSKWH#Rd?*S$*OCLyz1Jb?HbE@ z$gA!eq*_Y>Mrv7b>9X$x|d zUC32Rtx8s1IaE}6)sbhn6Y`4a5n|6Hueuq?NAUfi&yZK$8g!uQZA8VS-;hU;^<6T8 zgORsU9(lW`As^M-khk#&DxoSPQ7P$Z#aZ@-*4z}Rrwp0lXfH5EA~TDuRijsYaMhc@~XQ5dDZ11 zue$q@SKV;*m{#3LmBk_Qm+;|R#lEiwWO1g>orHN*9mz}dmz^vgk0}!Z;23 zsGhV2x!&){^$uE{eCk#X`A)Dp^5<`j(Fy9?44o*oK(5jOxyt3p|E|`J$glf($W|&Sb@)`Ou@=UG_T8DhMv;#F)y+Uhw4@xSEJeEU|_oxQ)bN5Ne&)C^$ zhF0R4$SbiE@=Ck|`5F5WbgS;0rXxRNe}(*vy(S&gcK<|vCeB*NXIbiTAo4SIDRhVO zDy5~_sEGzBuR+M`f^Jsc6UgVnOUPH?xoCh|-$%XzuR^{8Z$`cX@3)?NALSi{21_N8 z=cNMjywpVZsr6*!<8d0gU$NH6Pa+-AP{neQug^E3VT#?39+vWuw~<2L#z-_=tQNKzuUJ#`tYT-PaZ*dxqi0$6z!`c@LuJrALu_JcHch?Q~3I{yJ!DQ0bqO*5iVjpd#8vOXRiN8Tm?h2^y?= z*PxwRe>Whnzy8SU?*rtc{|)ld--uq+NdH7W`bB?9j($lrMXeQ(*Fk;svSJOD)A_aWrIQ_?Z*(fpuqgZ>UWWMh(733)bZAkw&^2`7GOv9#D_Jk*p~cE;9P*l@p~`C;@;ad<%DXh=U5g%6Uf+<{4}GG%p&@Ss8m7Fl zA#W1;OnEax-rMLQ(GbFJ9cxjUYer(`!%#&t>+=Hmp;f>i-E|imuZ<#rMd+FI$IxQ0t$_Yv{l&oB@iJLK~#RkSkX~u3QgoR_m$A z*NFCLi(;M8?@~A9zCDrq4n)32j7hip`V#a-(3YTLzb1Ld1f3q#75Pr)O62qRX5L^r1W3G-oR}GQpDhGLnE<{C@ zcL_Q`x)OOW`ylUSe{_&qA4Hy^XHjv*o<}95Dad`_K<>L3-K`n=5tUNj7Bp9{;)VX? z%B9$WsEkwsxyq5qRq7(oOJn4DX^t+_yqt|ZFI|x5LedKvr zhCDB8k>_OxDyO_cf9;obgj5uHREHvu>S%PNTI(Xu%NeMmVl7c6=^W&~7b5q)4tZYg zK}Rd^A>?^^7FAX3c~niBf?VYdls~)3^RfbYUVcPi~M`y8p!j~1hrJ&8K||CgFLEE$fN3k+N$+BWhq*yZLCGT72AZamwrdy#y&ffZB#)1y>kuZ z^>RFVOzY)jlR&`9<88|B|s?V?qA zCDYRA%cCciR~6;oRi&l8#v!jc8l}9pA@3@5oAM^1E?U`hkyrM|=q|N>fqcgQf_%pB zL|)mYcJs=lys{`ys(?IyHIa|e$>?siHb-9Bol$?qx}yQorO17+NA7zUx<{?|Bj3L~ zjRq<<0o^OTfZTTma^L0YnAX+rLB+BP=f9ekN4^SFLq4i?(YrdT4Ums&bL68s1a;9| zjYgiUm(tP*UPqp*1;}%?1PxZ}*T}QB1r1SbE4okGi9GrP_bHq|`oq!vYORVqYYou@ zik+6ulbR#boo{PNaSE2{i+6Q^hhoE7K4MPt}!;wcn7J2kj&?9Pn19{Jvphp$^5)GGD zBKKW~+;{&%$(|n_)EIfsbCCDE1M;4CLqBTIdm!)m4aj>w3i(W)hMv-S^d<6{ydC*W z-haQsS+8hRhasP#mCz*hsEK@r)<@HnmmTt2qUV*@G2~rN>(MOb{T}kRqs7WQaQ|d~%b?lHI|_M!8=)!68;rW>>NpL3uGnnkE5jz_E5iXr z3ujGGmHNn6hBMJJ^=OTjOYM=*?u(Gm?yJ#Cwe~^2G7LfAC^igzD-B2aR~6*GuOV;u zZRA&yFVOeu@iqEE`X0IOCgi@mk^iz$i318}b<)fm+8S z&(KVCqGI#VNz!}BeLqF+`xEkZe?^|5A_o=DI$1pqMh&Gyko#6d?pqgmhE7ACp{~d? zbS3f(-HEDehWaDV&~W4#T8=zJ8^*dcS~mn0JD4@DSb21zR1JAQ>Y^svj|Ry5(Hwa{hM+Ea z>Ert(I#{zd9<5fD$*6>4vqJ165BeGT-xcN0 zFpS_{#LuG9{m92{1oClv0eKs5BYpry-Uo=UI-`$~dwhl5WB(G#T-6L}g*;bXkmu?O zv{oa%A?Oa&MzeM|@~l0C@{d09tZhK$G=e{nXKg3q15ZYEV96v_0eRMr4{CzEN9}_y zL7uhV$g?&9dA)pwJb&LH_x%I0NZ8sAbd*kj1^@)q*EY($=y?Z~6rw>0Ok`j$XGqYgtg6sr%-_k#+wFo}`CjD8 zLlKLFyeEXwW9OUtJKptN= z^teZb@<)a8M}<79kC8{UPx)j{k3*i*-pJ#7B6)GlZM^7tM?9^WYB z`I~^oX{0Y8k90QjNVg%6wD^(9NRL9N=tx&bzAI>meBYFVd@pbzYNEVL(5cdu$g8*y z@+$6+nyU3dc<(P@f3kFupH$bH{H?)!B*rgQKQdeh7QL#ytbn|i zb&&V6HS)jLb{+Em-h+H(9z`uQf>Fr(`x5g0&PCqePtn=RTaH>v-yn~91M--+qSk6H zd{nZ(L?9DuJR~ym1od-YJCZL z#@|Mr75e~nkv>N5`xSEEjmRU&s+f$RB=QKVqOR&&3tb@9L#}cva+Q|oLbaZcJc6rH zH^pv17fH7u_q_+X@8ifLn1nnBGm%H|Q97o+OVfGMa^xzjk*oZMdZ;z4QZj-v=u*Xw zK$lCEko(p_?wgG~f>y{YtP`5A71kAbh4n;UVUv*8)m-Ft^)c$HZG3^euGS*&(N^R= zDpr|$R^^pOy`-|pqpF5Hs^if$YCR2kPCKA$73+$6OWl$C_CoI44|$L7L*AoD(JJlH z6Ucis33-p!A@9*H=-lEg|O@9&Yw`&%7t*8bK)-rrM@ z_xD!h{T+(DzoSu}Mlb<+e`g@??*iogU5V~i-Vdn1v=({He;|)}->QYP2B@_J^8Qvq z_b65q4V3C4_icjQw_7@<{p}m{NYFFL`}-pD{=SaVhkXxk~MHtJceD$W=O`2bFg*nyc0u(J;ksLk~&) zkgGg^T;*xx*?k^)c3(v$HM=vBXZHh?KfA|-*+u!YiyqcCPD1&!i}Gg|<(eNIcF|*s%|ee$ZzA_ygz{$><cF`!sZbMH={gA6XfbwS-<_u{q8>BR zbj98ev9HmsUb{gD)l6cCBX6xrP#xq|aw?jkdS@c9lGbRZVwWSYmz$B-%YA5;S|38Q zrN@xx`B~(7o{HX3>m20u@(G%w*fKO%`Wm_KdgQ)^YbEc*ltXVSuM(OkRYUG^JaUf? z>6qr=+Mppp(R)2c^vXy zW+U(AdB}U&6D`!%u16nAHzRL%0P=Q+p+#za3VAPIMjt6Q1AQ#bLGHT{x$jTNd$|oQ zR$kU|g|n7Og^_z4g52YTbWD5MHt4FLdx9QD-pkR*dpQ-Y*Y)6I zsaue*NZU~ZU6HctC0C@9$XBE$$XBH9$oCP~qdzsOJJDa#5VTbqjeZ4CMY?NE<>>nB&F1JF*z4hyk5$XBH1K^FvFfxNXFgYHDWBBjtS)f6EA|%hQT+t@sQ!eq_DSD){EYUIwjdvwUC2k~pyLZ??W@+Z$Vas{+E1|)P!Xvi za^EwN`{p9A;+s%W<=u`Bl=6^!q>y{Ol#Xc~d=T_w(C(mvPe|55Iple%g}j%|koU3^ zDyFSnh>A-WBX9Rw_zy7P)UV%DRSP-+dGyVYuSl(tuSn;ij=CaUfP9a1HS!hd0pu&v6yz(?Tj=S1 z)92+!=o#r-w76b6c5%bPS?@@_(Yw+u=vn36i^fTxq4(5Uv=N_->e`--9LqtY6}tmH zryfJlL}?hBBn?OXls6W6Ugn~=Re3RbL3!Vx$6bHc3YMAo84! zMjq*SG(&l>qFK^DrzWF133*Ph4Y8Y%NBTT^Lp`RWInrD-S9%9|q)U)T`ZMxKccM3y zSF9;_Zc=6BIh}w!($|nj`UZMSc^{&;rLw0bBW;5`(t#nCLLTW`=zaD07%h-KM;}Nl zkVm=(d8CE1laZD|3zb&|Es`1{k8}?5NEacG^fUCa^1e^!Np+hgBfS)Pq@zM?9P&t) zp-Aq!n_w4^xkt=x?b$ z+Af`hJkm3eN7@~Eq`lD&<@G~5rALuRTJG#*q&1L7S{Ln7UQ@JNx*vI@^N>fnHN>)7 zCL?WtvI?cI)XmVoQVuF4wLuFe8+oK> zwMj;L9`Z=Lq0-9hg~~`Xkw>}_d8F0bCb8p?N7@4&t{&H+veM0{ymUMANCzX2bR6R0dnjTT7SB967M?V8KP~LoW zlJo^?C{;N(8EI$axgQo{BalbG2sKuZ<>(Y?6*^V=33>FtB9Fd!hh+2>(P_#%4rNQH zqGr;$$fN%fdGxE1NB;{tU3oju8PY)=3uiT#IwQ|>3VHPJh1f^Pqu-}f;jA;&qd00I z9g5D9$|H~d8068PfjsveQI7I1L1#;Sk;l9fdCbMmOU7IpwN&2GsFidb@<_)ak91{- z{fIo$vgh-8lzLP{ZKT?$tyCX*q)m}W+7)@ESEF;3cN=OaJ%~KgGM$r=RzaTKny9_< z8liKgdyz*v8+oL^h1hoFk=E@}IIDwtG(jDuGf*cf2YIBOkVkqw@<{um^OW}xI$xTA zJknZSlaV$?9%(bwS$XYH7wIYFk$#Fi(gQC@Vuv7)v<2#_9v#pHQde}L)E#-Gy^u#b z7PT_Qb-E|s1|mr0wE_o!O8 zWRJQb$1X$OqbJZ+>M;RbEloz(NUtF8(VNJ7^cC_RZ9?PpB=S4*lgNG-B~K!?k)K4) zL4Fds7;RGJYtRh>VsIdAjL`9^<$W>M% zSJ{Nts>jZtG8ZTBwNyn#)weeCdo9h7_wpQ+r@oh>SsKgL=m7QTizX|ULTjY4$WOSh zAV1;0j}BIkk5F;xQ{<6;i#*bwQAxFKLw>?NxCgJbij_f!O68FIR!8pJ2>E+>&CuBz za|d*|dUQqQr0&Rldm;Djhy2u=LY}K9k)L`eBL4;I*OC8@^{2?cU0;okROMgLQPQu- z+t`h~jYBTsnMtFnjQrQDj|;IgkdJid5W5q-p&pN;%BnmXRh6Da-o{kqZ7fFqyFFhc z|0aDss;(Y?p*MAm4!D$e926^w>PUwo*E<@y-ihdVwVsaLwLx8i z?)wvR->t}Nt;prcS}TLR){a6ObzQ51d|f*M`5l%Ek*{n0kgsbGq1UvvG05{g6-`%c z2J$>FM4qb^$aA$3%~akWXtwk>@|gF(A{lcTG*_*ak>~0}G*7W6=q;%ka^JSd`+GU^ zJ8ie2`O3Q+eIVV7+~X1C9w~r;&tLf~lRc_~yhqK@VpYyTpGs|zx6u`O z8&{w&)Or*09t}n-6?+hUBRz`T_Zj5AGm-b`Bg7y4;5xn(@dqo=a^xPXk$Y@IcWaM| zU&S$3-Vtc7)^i=ikMHT*0P*8?)Cjps3zR?RbWBzH2R#+^0`k1PhkT@$ABqL5E7Uk$W^k?$H`~UM@tBX*PzU^6D`h9Vv}P{+GAELcZGmj*e36c2rr)x+dAi z!N}VvBd?}xbb|8ELJg$W$UV9s_vnQ@Yj+^e+Dqs}^>_`PB)yG1Yuk}$ zt;}_N+h46mpoUT<QakY{Z)%D;O+<@rJqfAt3WiarPVioO#0 zS608FH}p>Q;k}bzS)GoK%uDy}j&|~uU;Z*6+9l0F?`tgIqaBL)eQ)FnqRREqX6aPqz3hPASKbw9yILPbzUO}l_15UWLbpk4P%9m+Kaqcp zm30&EsjA9>$iLPpgO?y zrI4#sLys#~8;y|aBUfpPT%{u#rPd43)6x~lRc=RP73+`2NrRB9Jc?XpGMcE?naF)V zLeDF<6it?vBllg6+;=ycs@8*;c=xS{rYTk(y(-m0?%NQ#?^S5JTKgdP9fD>mHVn;{ zh9mbKi`;iMnxoeF$bG*;Zz}dZdP`b^-1m3nz6amRHA1cBko(p_?8>|V4;x*xgk2;{ymp^w%28d@T~gPi(NgI<)Osh{APqsT@)Y`6v2kdlGzq!N>&R6W zq0MUj4E-v7i(KV5^qXRTqd%nG$W@Bno>ZxZ{!(jwg;JTF*pxNbQlUT!#87)(hP&U5{Ml zF61hYqJe6C5)G1`L#{F#-KW^wC?(BDuCg>8(-m_)8m87Q=n-ira+OkC&L2~(EP6t! zfLx^}a+PfKq*~8HPe~n+t6YJeR_t2zjC3P%l|1AskD+JP`V<;3O+v0R2R*0QyJ(X1 z0dkcukgNQRUQp|AXtJ~$xk~ANd`m#F^5|u$B65}5$W=~9)6{x4dQIwtT;(b>U9sM1 zrqlj*}iat{8 zNVHh0j9le7Ra&5(YHfq|**|?n>ViC$ zYfvG@ZbJJ@w<1@$7rDyosJdF`pkt*EkgI%+YAg05I!;=LT;)&XD#iNq8Jb#4qXtq% z;b=4yQtwU9nU zuJSF)QEWA8DQ!TmvK6^Xac(Gsg-HdLMZbz;%7`e*psE=Cbpqr%+kgI%+Zc*$<)K^-ET;)&XD%H8gy-lrk(Ct#A zv~=Iz2Hl}pM|79eB`sa4FGsHO0P^$dv*>F*CCo?us~{VYpUeJ2lT@Y1y~*=ON%WI? zlt+FptBhVyUcHdl7_C-b^N`mHO;%plkasCsqr7WFULQ0?dH008p=ho0MufbXXuR@v zpb?sbB7?ZHDs~VWrC0?tTd}I>4e40qc{vGrURt0h)T15Rpy%XH$j`}7AirDt67tqw zN6S_39kg0~H>004nx1z7r<1XaU4@2K7_BdK4jY1y% zB;?W0KtHH;4szf3&|})hQuL!*zeE0n@(;8|u^nigw9k;Fa&hF!m67-NShPWTCn4|A zCCK}G1Nud+{X*+}Y9#t*0O|A8i z`!+=0-?PvTwRS|_-z(8B#jZp97ENF8Z$hqoH*)1+sEAsJBUgS76;EdMimt6 zj*gNpMXqu^a+N$(MXdvo`=*e`Hwsl(>m=my%|SI4dl%J~K0vPg1#;z|(D7nOJ0S0G4|KU&uM4fyk@xpK z~YORFaw+8b5o{X+jYjfoN?Tl_v ztUJ0uCgc{Q|l+leU~GTZ!OAG z>mSJDJ75^Ut5mEc8YCTxT={6^%Eu#*uL&xBXZl*w40*-2M_#c{Ag|cz$Sd|u^sXu| zLOz1qkdI(d-hg_~+eL2ehI|CCMm~bK1`R~-tH*uFdpQctSL|7|K$?iWAFmw^B2E<&z+HS%8Op}*BS5V`W>XqRH6 z(Y^q6wdpCXUyJG8%AHzJQ}-{E}Lrq}_fgj5{4@)5|D>!Q+X zZGe3ATcN`hI~SFe&PVQhDRSStP(`&4K<+ydRaWd7R8^XQ+;wCz@<4e?Bv2W0s(htaeHzW7m_i?UVYTX~XZwcgMc@#QZt+kMk zsTFeN&d8OoK@3P$Xh!VU9MPjbft7Qa=r7A>s^L= zs20*d%nl^b&IA*~pa_qZ`$_6nRCgMtv0f1>G$DirjZM za^J(A;C({19)aAq4!T{j2Ivl{5pv%a$bHX4etpYD_4WF88S?8}ALQ4!Q7Hc}Ormvq zLi-Zs|Ak59XUY9XQcqhu4EeLHN@$CE)I@%ktdE{lUUtZ9iGEdH$B=gcdP;dcL*7m3 zH|5>_`vKMQ$Y-c9FeUOqscm*vRwQf_oIFUKLzOH=f|w$TE4UOFSs%azFU zatB(ZyaDKAX)yArh9i$^Ec!&PQ;_H79rT%E3(-<(F>>FP$bB~<&&z&K^O?2sN+Hin z6|_vTnrOLH7r9Ckls~)3^U@W0UM@vLH7{2o&&#bS|BOfZXFU4dXFSS3<5B(@KZd`n zc3}GM;V_hc#-klRZaHesJk>8xylR3 zRo+LJ>l|Exa+UWTx>#C`+~YUo9>vEder4Z2iSenOW?>yhjIgjm1)RT-bXo_jW1Cz<$Z&$k$yn#u^G8X zp>fF^ltn(OHIY|I4)SbtMxKpc=z7(=5#1>DMXo#$x$?v47PUT&JR8%{?TXDpcS>&} z_g#eCcMbB|*pBj)x9@n)X=#7t9%Ybw)J6UadQFgL@(Sdcya{9GnAIk8p*A%#SIL}`N`H)%xlra^PgUaczl?=a=})*bXXQo>s$J$n*-@LD z8j}?>-+G7dVmIc9mM(m%e^-=xf7t8Af5Ww~ zz>ZtoeN`&`p;^WQoSMg*4wK&DJY1H`=bymg9=-#38c2X*}hiAP2Up`XpGG98` z!R)BJ^Sr3L^La6M=P`?KSkxI&kPj)j{2NG_FKH344FBSxM?D(;4?lbbcml@ERMh5T z3Ps&>#S|*A5k(c+!*53Y58qtWoBOGpnAe2@TPktSI{(N)wTZZY?_D4-8WHEptcC^n zj%4JUQ4ue95hG9EWJKM&){A;Ou~I?3QpXK_Sk!aXl9=bJxHonR?93~$LVNn~ML&bx zpK2HN9G-a{7u;M$JYz;ZVMjh2XJ@?r{6AhOdND;k(D#mdFiHPy^UTL3>b;m$J^ltj z=32dD&Vy7I-_6KaE9O)v$PfS1KZ(sas8nuL*5INswU)$W#Z1ElvLlStf^61W1^mQhD9BisRealqCTom^^JP+P5;|uneRu_|MF7itn@GR zGiPm$%Bm4{XP%om>%ZY<^51!K{qs)uy(7weHvKE-DE6F;&Es9%OwW|0Wd2fJo6Z?0 zyiMPXDX9@rU(HMZJEoay4UWoM5R+9vb5t-hePza7k2x3xGM32WmHu~jGYmR@y{7Uq zzjCo;b2Ds^&-3DLFz3a+DP1(%d)sH_@MC-#V_S`(wPI>(%{sTonV7G%zfzw#E?&3W8AVS6_hb%(llG7IpV z2&pS$-nH1fGWYPx>>YLUvFG2SjQhgE)R;Au+8p&lQ!nbpp?A!$>QYmqZay|=ehJ~m zuU^aty?gk)iTgTcs&~}H56gV(zW0?>f4wB?D@X-+0olW!&HN)@Z{y}5H74^S=3I{Z z?n=am4h49S_wc3T-@3V&pDPyF4`d<^Z|o&8Hji7qJ-qI+Grq@CKqLC+PQjj?@yJVA z0seV`RCd&jP+ruH(7c$BDpPTP0VF%>8*h1sM7$ALK#O$GpMU?z#TWG*(3mNYd$3s+ z^%-)>n0G>B1{HB#PrsKP^Hft{n=7C~|G`Bz;scV*uMKfOTS&!ydpzpBiPVUgF{QwxqMplew{ez$*K=2_r;h#%nB_=DoYD`w|YB5=3sz+qSy!d2Cy+zqO>XkgVpx%+< zvNSL1g0ng5f>SB#QqnH!QWE#KIHpGZGGtZMeR9dDOG$RrrKES%-{u<=b-`H@^^29P zs7pz`s7pz1)TLzDkrAgxYF^A=7EEo9y5Llbx|Fnwx|HM{9W(l=Q5T$5Q5T$&F_)55 zcFY}4s&|c;(T_PcCTmH}n5?YYF>&IkOIw2;jU4xjc zyc1)xrk)g&wd&-Utdfmlva%b;Wc5BJCTmQSn5-qI#$;uk7L!#kJ0>f)SxnZj(_^yc zoe`6@xp_=hr4}(+?aqqH%FBt#ntFCj)~c2h5la-wt zlhym;n5;29VzQQ85|fp6Sxi>F%OkS(K2D_nk&9Gn@8g93xsOz8@8d)=6MG*gl9|~1 zIFZc6-p7e#CiXr~Br~!1aUz+Cy^j;gOzeG}NM>U1<3utOQJ-_h+>jUG2Z}M9i~9^d z?hmg-eg8OSP;tMW&x`tuqrkp(%Y#JUm@glvCPrn&{l%)SF|}f*H10U(#(V-@aGQ%- zRxz84TQKcn_AWIn>O--pKjD$ejrou#W(T9bOBwg0lE^P^EQy({J^hE};*L%FA0%Fz z#ee<}|E^^!?$0}>YD9ftnH!bWF6QIIh~HF4{_}22_KBH>xG$X);9msU(`#l-)cL$B zX3fNW>$w0gMKKqKf8^iWnN=x$@@4*a8e{YLHfY4Rj5oFXxYw{KKdR?O>PYFJFxKXV5w#SDG#4(cO>fAbMs67%iN zm>rCJGR=)zR#A7Csgf}-%5i_ECGz`e^`dU%ddEy@YDCO+CgNozRWEa)|KOFhz+M94 zp1;Qwin%5)*)Jw*bJSnuh`X?5N8J|oj#_~ceR_OBaPi@Y;x|99f zJ2L-yrT=8kq7FjFqq+A8bgWwPznsAT$r1SHuG)fK8+%{x(r0IKrtF>D|8lxyyuQTU z<^127+kfu)xp!oHU+=0G%^C7vUwe{sVDH-a?_L{wU+?z5-gSE?XGrV+|0((pe|Fw` z<@nF99RJ)oE7<$Yf9`(n9oc`dY9qhp7k8oB{+>e8?H|Bo{XgQQv_`t&RGUN_NzjRC1#}CXD;bdvRYh%8mM% zFfZz3!mP}nckmRQ9rfXBUev6PiJH|_QL`HNDO%h=nN&9JkB~)WMg1Mu)YO<$BQ-4Q zPl8R2`lj>dsH0sk>NvKGI*!Alj$`krPebGWd9MYTKQrYOD&hmfJ^o^l%8j|A{+oy1 zZY|67|NJwZxXb)M_0=x!w}7jnPW-H>ui>oC{I>vOKd_FwacdWI4T*U{{~uf^ZV|`b zt89+B8pq7{Kk}zT++0NcQ{8)aFg;T-r(P;A^OHdI72*E4f6O%E=oH+`)|jZ9(N$5W zP3GHx0vmUM6-w>l2`H!+?Sh+txNm0WX8sBYuXwqcZ%N`#ySS@)+!s^}@*g3o6mvhE zDjD-k8F8AYqF&Wgao^^h8gmXu{^L_of3LdWcCcRNi!c}T+?Y373T#k&3uR?K+1o{} zx!zF^zfo&0wJPS}x8N?G1^MdnkNkFj$*8v+vZ7|az;-Y@=6X-?9vjc{z&wbYMx;kHNOa{md)oTuogWm6aRycIud@ zTEn8=p;;1hYn3V)^QVJS4P!1l{MUt?a!D@7ff z&QXuS|I|y(jHpXZ+>dk{MqNa%jLMoAm9;7+D`x#gJuSr)iaVd2-(>t8(d zRBq;X*5f`%iuijDJPVeLxkF5~i<$M*;HVeh$afi{KJly+^U+l*JL-dv+^F9&=S98v zPK|o;&5in}l6psFZI1ffyUdxxI>ib2$k;@(+L|3g!Ic){%5 z+`sqTl9*!`w|9H;4LMGjd~=`i+UuE z`>bI~(@Sy8gw(Jv-~Es2k6{OVs}w<^KzlRcUR;@AYR#y-%4N^;$AG z>I26aQCU&n6HR4h{!Yn%@LCb^-c@eYbhV3l0*w0?ZVKp{{2$ChfnB@z?00JlXrkgC zAL8x_3u-0{XxIK5)<6M`yuc1h^b{1>7wikHP}H}fqApJb_crR1n3sh;yt%l)Y#uYH zN||3=;y!3DsG|}y=(uhD!>&xc!4>slk*XK-7?Y|L^HJ|V?b1|W*SLRVq1?=OqIprb zR=uNct#YH*NxPVJlIk7v;bF}0RsPjGn99riwhX6n)GIdKmmJJfOCmWlg3HtL7o zsi{%lgUO0oT9 zDanqxdr9s2zn-xm>Ps$Lqq6Eny*n89J%nK~-yJBhdz4h%Lv(i3mA+oo$yl^v%;KsP zb(&SmTr2irq}hzstkj$omkRxJjv3#kxIgo~4JkQyzco1Xc@jIwsJo8L<4Q%o&Wy=C z^7vPys5hjd{_4>G{2gqU`9^LG%QSP=yr?6%IqDjn9d!hAV?OFiMf_!)S=<1`{JW}A z=S18ub7L0Po_tg4Ke#Mpyj-PHy`$Db+;<@IipH$IxW5E7Eb4pFd71Cw{+H9mw=S~= z)y(4Cy{Dm(QjhexQ!5=C81zEW$3a`rOy!kf?ySaWovL(0xoRDNnn;t-v(n;__iM;2 z%X#L!rpS30qk*b-5Avv<3tAYIK1csnUTMz5!P-V6c&wuIKhdH3dFjpbByRPFSf<|5a-H?&SdyOp;H6;a-=$os2* z@K&u&(_fsHE=S(l5ai?bV$kBCUxNvIq zKJwOn2`bIi#(f*13pJJtgZc%H4S5SfY(t2Z;>F3^JsI__o!)MD}%DqTY?FX-9O`ay{O9Aby?UVy8dh+O4@5W6eH#)a5I z6JKb+Sr=u&i!j_;uK4@0Eqkt;6=d0UZJ_7O)Ut<8|nfJ;N{z7U%l^hMB)po+XZ z;%%InexF3!y*lV2c8(^cvq$J&M1%^~(gh`kwN--lSyN=e^($m8pP)@ojE4SG80 zy`T+2r79;qPDZ}&c17N|Jmfo%3CMRIi;(Xqens9|*`u?vR_l7u6ghTr(4e5nL7xR} zM;`rARg!sWjy#h+Lu^=xy&hsKLu@zlcB@rQs)st8~&1pLf(Aj(XS7!C67s}oPc~}x`fyr z$W_LM*aGA#8$xTT8cCG~DF66|*qz8#o(-`NkgIG8I^@_SuOagOUVvQh?hqRn@)m~J zX5@N@@j7cYLf&q7Y+{IgjND^OXg!>{bC1TzRnq_8v3rqwyco1Nd3Yr}BSaY>aH$bEYuU$Y+$ znt{9~zX{r}ZgLcABDc0d9?Ol$^`1bk_h!&f$UP3Km$cSLzP~y@=uYIk@yM%q0dnPy z$n#gKeiAzwIo1_sfrSicY(8)6GWYylZXWw0;<3TY?U6lJq?ld4?_tx*vJvyo9{2mLkvi-^h1q6;4fB z&p>Xy3c2-x(E3VfT^3rmht|qXlfGvm_w9+i-C?2i_0YOJwC)bA)lN(L<{U zqN!RN(?jdmp|wzU(ziNt-`2=|ZwRf!L+k9&`dw(l#u@-g3vT;-_dNtNcvJ+44* zO@-EJq4kTc=SKjT|M2^)zvjU-91vO1|TWb{YraNY3qTcqNczGD8oR}C8`~2|I0}KpkC+K0E%-+uxyO^5?qeXy zIZ|jXO%BkCUNn-NsOVfpH!FG$NOJzS(zZRpj&z3rNzNw&$w+r0kjQ1BqL&nXt7zvD zHeJ4=Ns8twx=+y?ihfa)HPYsCoT5ra*C={i(T9rCM_Jwj6^&AKwxR`!o>sJ4QTNd{ zms~}sDw?C{b|CQ&UR2sQKq9}LPPDlkrKm(vy`p;+{Y%k*740>~<}yT4g`%q!J*Mab zMO%!uyuB2SRCJc2KPmdVqAwI>o@8@5RM9DlW-I!OqJJp*PSJ0EYjZhT(PTyQ6y2}r zO-1V!?fpBO%khd%SM&!(Pbm6G(UvD$-h&j4R&d)- zqL&qYuV~j(Z7u^8l`6Vi(SwTKR@7;n<=t1&P({-fU90GCiat@)waDhuThSOr=PCNL zqGuI-rD*$No6F&fiWOa==x#-?D*6wQ^qyVD+gy$TlHM~#X_qN_K+#)@oC%h9A4LU< zsuayv^rWJX6-6f6TyhkhsOVfpH!E7M=u1W0mDpVJ6cs7DSkYaIUQx78(QcD$E&~;v zrl>*DLyF!})OoVy%~mu_QMID$6g4XPRMA#bY%YBija78MqFWR_r)Z6$9ZGF3{S}Q@ zRIBJ7MXxFPQBm|Xo68_YWs0s)v{=!5iqfW9-kypI6`i5zdPPquTBT^4GMh_3MZZ;4 zqv$q8FDUw2QIBahmm?KT1d<+b36SXh-9R$xzN+c|qv>`jx9N@nl5|smWOS}qbT5$P z{4XHM`M*lrtHRQT0Eur=q3CL$gAXJ>;xR?<0f~o_R%v;ADjEtTvD1}yt)j;@_Cuwm zpKi;1Adr-Al%lgW_68uyd8x*J1|((fR%Pd84hE83#sY~n&Ib}1+@i71Y3v#x$z_M> zmZLwA(8dEvezh8VCy>04dKpMc^}VLswc6$~07!BvRWw&)?*o!jy#XZm_(fy0X4qT? z1Iaz60ZFRl1t=Fi*kTYU_BVA z=nNp?xK`8sP0=TsE`6pg{eeJI`cXjAGG_yc3~tcar9hJNXFyVxZf99^Fp!kxcR<2> zK9J;m3y|?$B>{jTR*-rkDFC^}EkpA|i;=qp9rUtn`NTv4&2Iv}~*T}pdJ(K9Q3KQ&g?!Iz^3&K2@~UB{r8nipDBBU(qd! zo>TNSkZ8{KwYI*814&z+skmPqU zkmPr-(q2>Ak3e#d=;fC8cp%{|2NK?^fF#|cn(h-#m)>C09S9`pMgd7#E&!7JZq{@! zDs7F@cKE&J=&xupknmm#BxPBov2SbaFB-ew6_%p_NI0s1q%89lE!A|NYPzkiwCN57 zlKcJ^NOHMIX$zJ15|GsQTOhf|?pImfV}OKr3Xr6`3`o*FqUqkzbe*rZ>9T<&-AEuQ z%NanD-%UzernJw2F&^U|I~ErG~I50u;~T@Nm(j@B)=0C>jGKyk`Rm?*ff|USq%1*qyGm9Q}cWV?21Xn(lQ?_p`?C zd!5bYI3USohSKIM?MWb!%f~=+k8XdoynTR#cPx;kJ0D2W{YBHgpy|HWbUm)O=>`Ca z3?>0de!o}R14?@fNbcd>V0rfk65e4z!dnd_>8{gsf7f)YG~G5g+I0PZq|7G+Nq!e7 z?N&w4D_W~)yFXdWp9dtCzes5pEA1~z`-jrL0}_wyHw$boM*~SNla)44X^ViQq^~Rb zS<#+1+5BQawfob^UJhh@3#C1(w3SNR{AQc(03f-?2t_l2B)=P#whTye{#N~JvlB=3OU(b&$nS&nQVDZwzMRRamfA2jv}jr|Bn%ChC{miHhaDZywU;XMaP z(%q!#p3!vw*4XVATAj=VlCqo%B;L|2rQHrB_jpceYm~O_Uo7uoK$7lcAZd>a6)jZs zlA^Uha^D^Auw_0%(F7nVOP!*>DtZM-c-H|*eRsRlqJcoddzzvKMfU><@0&nU=Jks9 zz01mFD3GM91d?=DD(w-VRr^z4cwbSMyX{)5Y(>WdNiL@=?JA``3M9F#RJ8d$HkbVr z6#z*tRZ6>BX^$!G1Ep>8SDQ;uAh|~&kmPcPqU(XAZck~tRT{g^A{*NeNOBniB)OcY zv_C8DSs=;pE2V9JuchSz3Gb;&n*$^{-=XM5P4|ta+u=UT+aE~oF-~b0DQ%(BUQ*gx zrR{jXz1tB$QkDrol3$(D{;ITBl(r5?O1j$v77YYCne|{YkmNT{WA9h=rlR#ga<{!7 zw7JB9gtr_>Vz1KJhc)(HrET_*<=s!wa3JBGp|trxlJk=q`?1Du^|00UgMg%Nqk&|k zI|pcWPtx(5H1-)q{{}iFhhn!~>^MCc9R?)47!^YkjUzMAd%HAK*I5yrdtCf658PjE1~{CBBAj> zMy5bgsyl(CMlWgXw;H?ileWzHKq9M2Kw>fLfF!@a0!e=V1d>|*1Z2j%zgdpKKr*{< z8jz%G01|0D1SI*rqo{MEt$8+()O;9_q^nkRouWoXpDNmFiRI`6B+?iQB(geRV{cLP zjG})l+V&}%%VCPfDY{6}or+#o^u3~8ms%MN05W=hNb2^WqPKyhr8_ON(%2VB zq%jmoYBU{4YIH4-u26I} zkd*H+ASvGmKvKRfp0(xc1tjGg2_(6krRaK~TDH_r0ZE@(1teCW>vNW)H;|+o10>}; zTVoe!?9&>%8c2Fd_vdXc2P^s=kmNTDNb9^V5A$F470jsZZzF-g%}AmO+VNI3qfu|H|-9xvJ0fk2YWX+Xl;pt1KW?Mx1v`S{h(;~f7*1%07;pr z07-tA0ZD!jXu7u)Ij>vZeH0Zas!(*bqQ?|{03>a+#lLLJ^a7GHk5qIHkhIZ_KvKSC zKvKTXHC@IVcI{SQAW3%;kffUhB)Qz8vCnDj8X&pvwr^VA!+?bMWTjoGv|E+-ywbi@ z+IDYQ-a~<;ET;g8-d_YHWm%}PF9FHj)+%ksw=FFXNOCDsS}l;|a*xKotg+uKZRdAv zE=K@KE)#&nXT2CmYJQiZS2XrJrTykzo9<{J$z?K-S{#a4uBTLIs zbfThj72T|8xuP!>ZTGRwB~MY2qKg&XrRWt!>lE$wiOpr8qSJs@?N4QHQ1lRx$lx8N zb^g>!BO6F;_Anr^5!FD#aUGE4*9au-_L0VJ`G1ynkfPCw&QWv|klf=LrTtrJ+pn^H zt3S~0HB|azfv)UHXbRBRy$D?a^bEJhqd?Ld-vzps>+5`G(LO*@<^n}k8arQUPb&IA zW4HL+^7c|R5=icGmeT&D=qZg|rL=9nu(|XD65ih`twzyp8v7EE)MzbG?fz8X9amfF z<^c(=R#zeFNPMBGdqz}w z94B3IG(R&V)5*+)k~kk81b?Sf^_0@-&Nd+Bg7h(yiuvz+{d>IA*%^+~eGLjR+Kb9q zw3*}JpOa;X8IDs!=wEC=J6%!#?hJ#AWN{%j{IJ~NKa%p)Af?eqoto$v4O+~DTIS6fq zK@lgelMS)@fms>Q#~yX*7`_Bnd-@m#3wrEFVMz(wQu>P|24A5_d4PsZk5#YJzLPc`xx z7B-*7R?3_;378J)JGSq$$M$WsbwN82QBp`bPY9d;f+qRT49cH~?)*;*pT9@0-Uf{; zMy&n?QyaMpszizu>nkFS&00n(IBqE!)st#S(}x0G!sFa2Kts?LX-)~y%{+LJ7QM`9 zK2QP2-U_ribUn?v3uq|Q9#IQ|4`cNKy)?*&aY6~cR=I#zFnX##f)|WI+4>w zfyOf08|Zv4K_8%bjD`YDWR7B>Q#joepmB_*0SU+1K*BK(Xclwa3^ak${T1j*&Tk3O zB&IC~n!@NsAj#z|Aj#$5K$1)6&W>{$r|SkJx$FT{#x(kXNjallK$1%ykSRTojQ_s_ zk{O;!K;l_c0-ZmaWHke5Fw7H9)BzgC$Xe50D9N!tYkCDzA(2BN?SeGdsFLZ`gNmnD z4=665P*Rqccg&2^vWbI=OUq_dmGti|#*~!8Ug;J%8OLU3M4Y@#(xBm;Iyv2(E}0UU zkyS9PPcDYv%uKRe&fYr4ZwFF7DA(;uq>9Sw(u(rwI8ib)lZ?V+yAX8}7Xj8bGjk6l z?F(Z)ywmjRipt^T$5vHURKX?5%p`O5498R#S4}Rdo(_K_Gc&5`i_6L?&WzW~*$J_G zYHU@>86{QIONLiXEU7Zao*j$jNMSFM)50z?`jC4-7c6=W$q?#Ya|FUZ2#-P77vW%p z2O}JT5aO*VMR+K}(-DHF<}8GVA)JK}yfv32JQCs62=fvC1>w;MpFoH_YyO7tScI6` zpv`AB|A+86gg+uY9$|k-0QRkB4#E=u~qVW&1 zc3(va6k*@IyrSOtZz9IstQ_|4vR(X?qq#2ZoEs;=H@P+)!?pQs6L*LPktXgI5yepvcdHmJ zYQ3>}7(knI3%gx3W&Qf~9+wa_w(=3Js#oQ_P8^3 za0`L%V~6Q(AaR(U07~pI(LsPT*i(3LNQbFU7>9}U>sRP7k(~b59Ht^TO6Q{76WZEg z>T~4Ab(nnjZ+C~ucmJ>s(|f(hnaM$~BP0&fQR*-e5{F6Bk;5eE$YI(}9VSBJFb!9S ziI6xoDEMG;)|0GD_?)-O1^ibC~Yp zSg*r$57WqDTEr+hhw0Ol9i~GIZTp{9H#BnVIm|BQ=vLcgv4R`LLDYTzXFGe<`r~?iC><< z?P=x9GdshC2?_VaLf^*O>pE?Ed$#HASq=FjL-8OJe|xr=N8}*S=USwDjAL&DlBuG* zfgWetqe@!}^f#uhP#PBdIZrX|W2OBENZy`pjvimkZ?<*@`a7rF1L$c+`vA#wN?#zE zP8kLyZ_ma9JkAevgD$sh_34e8tT3*+06E&3Hs29ccFV>0L(v`IcDZ|*>NG{Pql z9)qwE;jsu`Lr8DWXd0Q`p8bT7-ku!+ZKJnm7a=?W;U@^`?b)#?cbiWJ_1nn4jqm>L z?%Vk8AJ(^7*~|Jigv7TQpuP`8JY{e48EAw;?3H%?aw;5E9>JwfZ)Menq}b zd|^=QeH%L&w8^(=?2xlTSlI+y73x*kj0H)*2BVQF^lPxMctmc_xA~T1y}r%&O!N9S zKQhhh+x*0|#J&yIi#llz`8Jy|O6=Qg!Rg4ii7-m++icD0yuM90rjc)x$tbaJvn{9d z`ZhZ-jeMIO871f23`^O!$qnh-9K6Z5+2Hdw{}aB=!5i7P@!h}OeH-8X!}>NK^s>GU zA@OaFRo{k?_%@P`d>ctezRiy6+Yl1pW`z1Sgv7U@uP9&z(PoZA=vUy|G+i0gdf&#* z+cbI6wQ1g_L#%rK6}{;?kVo?7e4^eQ>-CBHGR^A~^<$dXCpv^_iG89xPDeh`5sVW1 zM8|MC@`(mAO6(KGIGxugI-Y6d6AfdO*e5!H(|LU&nGHG=7H<^C(#lquQ#ujoM5dhr zG?vkLAejxS0Fv1tn%j}tpsRpnHt0`4G8^<4pbOb!x(7&RgPsIRJR9^pQqgP>g@=Xo zm<|c!G5t!H29ccp*Jgu?FdK9p%H8I(L5FN)kI8rccK4Wk_YdnaeW)H2A@P_7tH(r0 zJSIs;9+RXakEw@xOoYT^8mS%=A@P{LRF8?!ugGJHFAZwF$7E-NHhD}Pb2jK#sbUMd+J*MeQ^Lk7(nCA7E&R|+%kLfH_FUXSS#rjf^VDWk+5(>zYsoX6C_v0jhqN~V#=bPc2AJf??I_L%ZQdQ69I@|gao z=9o5^$8_jM_LzM4Z+DN$cmJ>+(?{wt5fYDShW2ht;ux=a(w1vnqeD`y%i|Fi!6g_ z-bK-wO!F>^-i~P>DQ|PT$adm%v?zKPMu`_i@4@M&fMZWai5Erh!|A+>qI)uJIdVCG zQQ}3>2XZ>^qUao^(Z>2djFQ_}e;hA+PGw_#-;l0NP8iqbSGqK4)5dz&2@5$J*|qWA zzujFM-~Ge7Ht+PZug?gHYje1|HiX2rk#ywRNIG(Dc2?JhkhnIZ)wLlcuFV>CZ3uNt z*Jk$W9Nr*}lW}nL7tZ>!o(c%S(ZiS1F*rM*{$ch_&$yq?X3EDd{+w}3yJE=r6v9p8 zlOlHKHKdS(^M~_^1_?X;Y6#^6Rr5J1LxH9q~~;d(&W#yKo= zN5gQ$PL7=a=eI#r2LBVouI7mWf~gsb_}vjw88K&PY@sE#Zl)!Mk3S9Lbt=}WZCc`E)L;4uA!!MrQA-G|eOh95ihW`x z%6oJRE%9YDEis~GTE!V914~nRsPkK_Waca@StspdN z1)&9f9DwIv3uxlC0G?ECw})1+C&lDgy&SwQA6?iU`N)ha%${?M&ZyG-V*Eo4cOcGx z51qQ1$$XjMgz4E5{!MMEd#@&XrAnzdw<=tdbG>R zbeRS9%AMx_;&NYzyA5nX-+z}#?w$$HC~~t`+lxFE`*NVln~>NSp*5nwLQ7rVGT-Ty zH_i3M<$WLSHmn7C@0d{D!Bd-2G@=SEb(#Obp1D`%G-nrH=7SQ+ytUJp zsk!JW$$VdxIU$j`&`9P&OI_wNJL#1<&B?`Oei!Z*Yr&?ENF?(n6OmEmXs@Lqfau zb1c*3KPp=~)_zhaEHLyPJImV79tfkL?TiqWbap{V@2Gc0Sd4Htg!E=%cZ4$$MiG)e zPtj*BLJ?=a(nvZ&TcorHlorsZ@amnLBuq>92x#dZ$XuFMT6$nsVFA6&AD9)3)w61Z zX!#su@&f)#b4YU1^0Z2B9cmZvt(##%z`4GWiE6$bepldkxxeMLcLsjQ)~gTX5f2%j zm&alGq>5_svN(lC+>X&7l1X;?pm zq+z77q+#bEBn_K~kTi^BXf^C*R8yYj2}PXsh!YyMg3z#1+0b@TT8bJL#+D?lVPadn z8g{M6w)`JzS=54tUF6p=`?YDWE=`zHTos*wFH2Ul7J2_n!thuvD$XRuYNADMvP+BJ z4%8xRrg;3`&{|ZXT0|(q-&YkHNm^(>8`KqF%d@oqZ7uo)H9EBgEt-|G7PY+&&4o{3 zRm-JAZ?&vLGMloYwWwURh)~4&0P+k5kmWQX++PYG~$p{G-6t?Mm)9AHKI&4f{mbx-?>+q%`8N zR5W60utqG|=o&FqHG+_6gwRMMghm=6G|~v6{cmZ+-%;+-E%*`JC8ZI)b5c=>p}{Kg zM2jkcdibAoH-s-SQMH0l#94y-3XLQwG|~m3tyJ2tODiHtX+@vjsc6N{y9YWGk9P#E zxD3;5;!h9~tq>Y%h0sVVghpB+v|pE29GH|=^gA>at@t6>RxIrZTJfUV3PPe4LL;pZ z8fk^lNGpW)>(Yuo$!SGyDq8VVuvRSV2wL%yY6T(D3Zao!2#vHtXrvWF`*mr>o=Is% z@03Q6(qOG<>LTIEFLi=@T#pcOrMZZ*RMUP;=;<1jP74y`u zAS7BLG|~#8kyZ$ev_fb>T0xs;<&~?vRE?Q;vC-IO%VYIRC@z~`#AU}0!#mdLJOOH7 z%0*WpYlrvRR&T-YV87_jL2Nc&=u)m_ z3WQY%D-l*BJRKp)p$Z|I*||Vxun9$+7nHWbAlK;t(Z0`eO(^1Ag*c(j zHz;7C@T}Dhh>XoU7mEw1MHat+>zg}c^>ecpP(=M46VAN z#qJd?&gI6FHlF=`4uMKcn*(&%~a= z`)<0$7h6+AEQSRR$O65MRyQD3tlMmQaVn9BDU3}85$_pH`K8C!ub{I&r*QK(P}+Rr zE$nvTHB?6a8?PJ+P3^e5yZH<18)NfEWlf=67lDKGkv={)F?Vb#vPOxn5=n0Ma;*Nj zSp5plqzN}w;j`baU%#H}L0s-~m_o?jvxaz;K3nM$p>w1;UjWJHG}4gETKrxG z#lPWa9ca)==PQKs<9%l2l#;Ttkrh?dwxb+y^z;d^D5d3-2VmDvag{j~j0$Y8bPJpe zEFI~Np0fayo1osqJB=@yTw2byXra=O9ab%5W>O>lnNv-WtkHqD%6u0Txh5A^PUpn} zQT+X{ohiHW8PkSTmrNt_X4I?r`aP;JT;Ptz628aT_OB`~pImZeRzZ$K5meo;_oZz5 zbFnyrI(lwA0%bGgc>aTs|J9QJ4oIC3D5WVapHNjYt)#r#Q=I*goT`Rv8zhe*+zY?q zok~5)egTc<+i=mozbOcJN0ngYOQuyaXCX+!c^VAQ=Dy$XOtwR0HS9%gS_C_k0m58E z+#7a>D!v~=5)zc9=4OO2%r*BQBs)c=g^JZsarz?s5FwUJ)TAN5gAtBEcqqaKgpgIu ztq2c8cn88fgfAe(k#sfmIbpirN`yxsB;BI{W~?(v8w-DOrKpT#%3N1b2<6*1FR!RK z{+o!CZ?baOh-bU9pj^#$VRxL)K0K2Kpf0GK*jz#pht6%Ke%94_!k~!rg{J$;pol|e zPk4tL6mia0+WAVmNogb*xyLG{tybDX479)DvfO1*bLSe05yTjnJ?@mHJy0O&w zCyt5DT1^TdBg1Y$u91OSl|}{$=VmRYzhz{&_7Tc6_WciSG(H@MDw2`RF-}=N{lS($ z^!U)(WD%nx$@tKNZ6}#zslpGjdHp(b$-lWpOtK6QH*rO2xbO@QSg$1mfjETj;h_;6 ztMD@};DjayerjV69uSa+z5C-*in_^Y+>R9SBJn^q zuYadl{nv)|vkz!^m>)PiP#cdsb?He64y_>@KNQl+geIjqq+LJZEpeK&7RYP&KLcf) zZ!&y6Xkc`{LwH4y-EVQ(STmymmCs(TE@(yrYP~Mtdqd_AmH*%`fMguZZ zCn7Ig8)P&%7WrQ<`G*+|dLsG5lDwtS;J--nz9h4wLAETU3q2ah3j=pkP+JF$2h{#F z9*~%5JfOwbG#)&Nkj4WlFVfUdfoMGV3?YpN5#&eX!B~Ve9+2J3Mfeax8V?>rNaMjf z2x&a{03nSBPS$Nbd$V#U($)aRB)+?#LypYLYeH(_VTN2X(x@C{TDB|u7xWNGkBn$KXF8LnbMCIE8!EBAs9XzB z{ml{fH^d=mO)4wW)xa_1kGj)Bp91`imUf0B#E(Dnz82wm_?;Be3uscifC{(T4Z!_L z`#wP-Mf(UvoV%1pMoMV3oO2h}(x^efqjddHQJ5y9bjMb$PVJ>yV=)ro$O^w|MegLh;$L>?kVrv8ogyQ7NI3N(pT{DAKO1QnYHA zP%Bl+=lCUIvgmjeI7rRfE$YQ$w~ho7yG54hHjmx99TdOa8V}lb_??X3FK8umh3Uqv zw_EgdNp`D~QLf^uYJR%RR~q>fx=@_R_M&o<9dj$&R45bXOPy&5a}bszJO*I}!Z8Rd z5w1npQIzffRM`m0wr8O=qLf0*LLGNwWy?0GmCDxM_Ul%UiX|Ev*KcSQvqPg*F&cPX zLt{P!ZLc8vRRoox&IEO1`!xX}*)PnLvi&MS*pXD~Q&lNKV!wn&_Dg7FzjkMpBKt+C z)hd;oqqP{fb1<_nzSq+O(I)boIh%Yf*Z^a*Xz&o5Wj(A}p!hA-iJ(RBE5xt!h8_ml z28mCekNeq+%HX#`aP(`)Z_Gvr%)U7pIrNK~;}N1=I+V(Mq$)#5Hed;j9*l(Mpyi^h zGBk@sC^?nEu$2*e2HBGT4g5@gx(jcmc-&Ik^1lb|x4x%d+@Af{gAg9=YI)uw*L3p} z`Pj`x@t8%owHKF1{xH5a>1Cxny^*ioj3!W-=+%pRJUR<4O@8(n2)iRZ6Cq9O%tQ!n zaLz_Ze(X=ZtRG7#;w)4ed9*@nV#aBHCN9Ywmn4CWBt8CttE-VyFdB^=e-U#nDE?Dk zhJr?q1%vUUZ@8LEWJw48*?mDOU=Ohccfjs2%R!dCdgx<}bq5Y*QbqNYk}9{jW3~hc zsca-}60d!XnGJDLd28_RHwa-gYW78VAwu|{&K!itAiM}6iG7$VKA{Lt^9t<&6Kj-w z4C|^(+9C87H)$eJx8Bkx-pti1fm$XCgLTy<&<&ugLse*mB0PB_v;`=U&;oQd7yEDb zMuHlAw(Vfi#8xTq7o&s;5Ak?Z-!fL09{;<3NP1n&^Y>PDv8^aR*VSgZt{wtxdI8?wRc5G<)Ln{4Z?E`y&h7gGR{L7Mc81}9hQ_|{9b`@Am~>j9EtEMgrgD8M_7Uo zrl4j9!s`%{7T$TF~*=*C;k-`3*d{bQ!)KqP4v0hP86B1DijYKUpGYi)Gk(Rd5`t{vyu|8;(5RsW) z>58o3hp^nc(F33G=-$bWs4uIa2+1c5ga&PQgyz@D+2>~EI@P^>Pt|tXkj;(DgyfJV zf|>auOAJvWYo>-~PZZ!I6v{jH`UMYHfep>+2;Xm&;G^KMS5v{~Aj)NEb}a*Nq?-ZLQLMj_IOaviK6VSB?#(<2@y$E+ict1i+K9hl}$wT-c z!a)chLKs8%2*ObaA4OP%@G*pC2%kVmM&&haKteJ$3k^APYzkxZ?Z~3C1)uULzfalX zTNJhTEvE|UY3}dyG|A%U<7cl_dd1UdeX-PpWWGUYjmTbTNt+K0J5Ba{QenwMI|#o| z{91hCNUZ0fQ`84klus&+51W2^NWm@0R6nxK&B?(kI6@I;0qP?(8tjCYwBRmB7Gqn` z#?P7+96jpLGAPwR6`jU}6yDQ2Z8+f_slp>9gO<=1qUJ(NT6l@HZgsQ58#1jD3uBs! z&d@^&ZkL@moZtqj;0TG<35~Q)Xhv|JcV03exn6mhv1_X+jH`*X6E5Vi$48>8k*|NZ zKRsQMr6-&CLR^~s9;@k-%K?4+K%^~1vRtFc%U43OzkBdt zXIZBA5W?*c|1d&&>R61BY|$eKX>{2Y|A`?aB(r2fTV!H&bV*vWFXh$vzh- zS>u*A5vq$GQl5obUU~Y5l~klTsh5=|A<{mdV z(eFylg5NW7F=4bleMU)Pbrr9`Jz!8(Ny)&HvJ$K$f&#V>*+R#*3YdXA-MOy{jMm;_ zM#8DYP=>Pts?DSXi7<+vy{Mcu_;tdMl#ebrd#rH%;0uYu5t5NtXpKlEG`EQGT`lrM zvF6f#gp}V(=f~!4P1zd3;p?J}Q)JN(WE$hYuS3RY;nw`|OAWp?^aLAX^wg~aU zwEh)zbN3|4kpe~Wb6rIV?iuE5!4BE3rK(DVBFIo%}2w>2UD zF2vth=|Ky=2>R}>tnoXbcvR3>-xow8vzAd!HKg>=d}mjd4n5FF=QsgK-m^{u z5|dvJv=4WknLzZ)C5>lvdosd)i;+t*5`IBK#{t zQbi1){C0zA{_eOBXw)^RKCUZK*9fyP)M>6eQmfHD=?T|$CAw#*?%2ln4D-S6X6s4y zqk2-8bX|$+*_YJ@_Eay*#9U_4Et{Rc!Pv+MMo4BH!R;|N8Zl|XATl6Q4JHYnnZ!i2&W7%sXxV-0G?WX+>|+*XUG`dhGCoZfG^NCRDl z%Wvr{&rJ7yi07X?`~_LZc|&E)b(vD>=IWv6v@n>@6O*Q+Jq^|Qu{trMUg|UX0tvuSvzrtQwQhkU7_7m)3t;i`}k8qV3CV~+rOcL$0c4D>A8 zJI$E_v@fS4E7gP3r2$bNn1|nqJ~yBqvkgWKYp7zHZ1(tr8BNa=Zlez|Ld^@G=lLHr@frkL&@`NlYro ziU3!~%h!B(wI}Nwjc4`=9(q_qWy3gALz$o-@LUZrzd4|I65O{cI70HmS!fGPY{VH@ zi)Vo*t16p{5b})1%)8=@ZV%xRxR&-UqTj~*>R*woUycJo+aAAT@H?Viq#$YM{_Mc53Q&K!t1Wtx7A3u9hd8v=P<&6n7UX}zSSF&Z^vebM~FIPUf|h}?EQnc_5{wfJ3*-!Z;H zggPB{sy_Ht$K^igm>H#I69-jP(J}Ej=Ggsb$YEqHs&5_a$#+(Fo1kQZR~^q@`&2lhye1)+Sted@~GMvmaur6 z1)>^V8yg)!KCw1-qMuwpWNck*te@KL;UOnas*9c6g#OTwQ)^?V29U?q#>NGZi)v#< z0p#M^SaB2b!$Zd7p5p`P6G5NYg#OTwNwu*_0p!WGvB?4CDYY?70yPmF`DfP7kQ z?6d&#)Y{n80CHJvtSo>$tu{6-fLvZ1D-R%7)W#|T$d$FRN+p|V*L)&yZ;sEriaQ?A zb?R!-rD|*Gf6YhxRUJL2)Yj7f0g;z-B>f){SwsBI((#wwC_a$S&4U zaVO`h2}$fiEZ8p8*DXLMb+z<=JgbHD7Yq#dJIRVkl2xHEt1gR8a_$C_oJo>=tS`B> zes#;iR#!{^$M3wtWK~2Y$%;vmRgo{NE-OuP)=NpwBuPHSpL~r;&gvk^nIy?8eaUAn zt3mp4OLOr5H5hf%oZl(BUePl^w8kgR>5MT*(!H;!CkA_=ouud%MXxIA3`bwm*^&1$ zRx^BX{^;9-`}J|TyOWK+!~ z*L=Qs1TzqIg72}M{DuttXfnoi6}42bWFEoSBJ-g8L;QSy^2=OT`1^;NSZu!jDc?WT zB;y9Wf2aw^4S4^4tftwnwj*^im!0rz{{Fp?k{C!2U03+_KX@bG|B#J*|3f$O{c|_+ z{SOOwe|nPm9GzXBBnZiqSQcz?SN&p#7Iy|zZ z^YnVmAfy9&ifPLYLOMJ(9)2k2XAsf>UB|TL1|b~=k9Q@VK}ZJ_$>s7Hgmf5p_Genm zAbR@2_Ms+I?ld;=uxMt`X)>;*DSAqeY5fd04~+^Tc;qR`+$SWDJngu6ntv5gKhKn{ z$?JoJWt+9cFNQtCRB6_dg-GU>L`b&9$+4|Sj%{TNY<3b6r=ZMUjYmv&2@A5u)^90G z{ox?eUqjOBVrlW%Z@Oh1o&HU?%>48?t$v7I{%?6Q>SUHj{*KiT!K7NOhj7ep#EcL> za}UfKi^UO9^K=^(X=*uQ^JeD6=AEAN2pz#7CzhD(<}GdjDK_t33LeQq9c}dNWw|)# zgWiwPwvzZ%$EBbg>6_|!2eb_Q{)t~by0*E*ufck%YD|~A{$MMDex(=j%4KKE3-M<< zdXbstAZKeVNLw?><7|~RQnn8T?nNXlCddJI#re0=}@>LrA%@a@^!l&-~pIrc6H>HhRK4?Rv_Vm$50;=jKmq``y! zCQ`MH6no_^lJpQ^@Sd-r^j-CrzS-wt%qct-QwjRI-|N| zW~`!YB8kZLM@aE(kG^g+-EI`xmr2toUzrh>t*UMVL1l?SWYnxA7P=s3p1a%^A!#jo zDQ{Ut#aMK60K_N|l9hczy9KF*=6(N@FILr~ZG|wa81G$17BXGonc1|~9y4M+YOQG| zgAc+^wXvPz)*bU*J7bP%=QyQ)$S!rUUE+VkzQd%`bf+saE>_KOqhha6BB zJ0Siy^1xX!v79)ieh7AZxW}a-{p(`=-SPQ#F;jaPsRq`?2D;;sUrhd<-B?KC-~!Q@ zgMoy_A%01NO*EDgaS+lne&DqV*5{ZFH|t5BY6$uTT&4)ZuAi zr*vjQvZZEUScR^0B;pRdlku~;SXPlJa!X($Y zO8vVl9C#TM2)X()yeE5v9@)X*=Lb@mzp*q(alF#_11NrJASrj5Y=pED)LhgjnyYVH zOG6)^w$z#va@D-;tXM7TN5V^OndZf6#p(jA*z;H>BUY?Csv%bFI8JvC?(SWt`5-9g z`xMk|#fFD#l0ci*T>JSFhqQ1n#?{v~*hNu(dr`Ul%Q>gwgnF`Y_u@CN6&5aJ*;`gc zUqzF(e%(|p3CZdDLR*1a360Kg+K;VY4}&&}^}D=P*6*j5t>1`}(`S@cmEaTQLDs;= zhO~bB!o~4fzicM?tlyq0Y(g#@kc9Q~N+Xf=o7{3&v-NFjX{hy!YRw6`YTnVT-<4|p zt_iSyx5FtG>qj4@PtE#`3)dvRe%^8m`4We;e*NR>+Y0NqsEw?j)jeO?TUF6yt=~SX zmW0Im35~3u(8&7rRO?4*qgcO19_#lcidW?uTHU_iFFxxxw4}P2=7pOZdW^IaB@1Z} ztM_kW4`(pRWe>&qo~R8#$YmFju!vp>{Uty`Zh!Df!epN8?0*wV*G>?@z7q8CzL}km~BIc6hp+@fx5+|UQ923J=T*{q(DXei-H0w$^lWVaLN2?JgcbEl zD3KMtHDvd+Ro`F|a#_)tDh)y|X>>Fzxc+1w=ep0b5>MlBK;GQI?(x|Lt0IG)9ACBbec$j)y!3a6LQ&!B&?=aB0b+&tKK9Hs&%)j?g?7S zmpY^s+&Qksp{-z+@eNb9f>!^0MQ>e2le2;&R4oaK6%-m-L7|ZqJW#D5p^ah%-}6|( zMDvoJ4@LRq;`c7(>=hPQPYE$E*;2BQRxt;=X?#|(H zel?)`wN!PNI<}Oo^%8Pf$wyThgj~|-Xjbw=wUVC%SV=k>O048kRHJ1pnP^_}`BpYl z^G2klR`YR%w3^4p)jYJ-JZiHP<|SKEg^=p%t9Wu&v{BWTkUUWfZ6!>Q(CCT!Ahx13 zFG*;lSkW(9Wkoj&X+`Y_mO$t>Sx6i4r?`629uIodUl43XU(p623q^*V$UpH&Kn$tmxaVY$l&s2-#{ru8>yr zfw-E7wxYA!+KO8J^i?6biYI49UstsyBvw>tWJQHWRy0SgD4~sFMW69l(M0o-^buln zd3{meEQnv&zUYuva|GU$`FzijOlsnLHmJY}x$Hy|R?{nyL{H=oHrE|}Rc>>MxUA%5 zT6;pS+IKW7`Mg@mmjbNh+0ZQUHR(I$EnCS%^OARmY?cH@!qJ4>Q-Kq5*?}aipjRS^tl+W)-OX0H%_ZWpg12bx3At+D(X8Mb zY6ag8u!7&I6}%hONX-g99sr2gwpH!9t58kf z)tVA=)wH8ow+gjxRRPxR(H*Ui_bIB;vUR%^)h_obU_EFN{AfpC@0Ob&xUc)#$(Jak zE&D32o@M|FvjQh#-ZkHZvY*$Qcg-!AIat+v6>Giil4WuqsoD{ekKzce5w#Fn8mw1e zwqEH5Z4~P@vsKpXhhXbv2MVuHEj6U|8nRy#>vbHHeAeqb6)quHpH9Mhd264@dc}gf zj;&`~)n2XFH(FCdu9|i<>vgVLuL}aK*X=#5_4*mrNX>fXx7-B2{u|sPzCqtrwwGtyjQNG~45LH(?l$AHZ}e zDEZB{j^MIn_hth9BHT#KF|w+ z#5Lx0Q1X1zL4{a4@JF0TTw~}wfTPPwi*X*A!jPJ>086NijEr$0@ADg!Noq(#EZrpmy*W5k##f|8mQP&Lt6Wue^F*&3nJE{>NBxmX$44HQ2 z533Q1I5fw1GRD2Ge25L9(6)9q_ln>XXXu-E<}AR#Gi}Vdap4Y2ha1lv<5n^VIk})YyMy zp1Z~u1=Ls$cXZWM2qA0ub6^cG(Has;t%jd_YB&uwJfsDUeZ^bD0R3@M>U;x4kw@o7|DCe#WU1=RWfJhe(>f1VDgRbknT$))A4x(Oj<4VMJ=))%#g zgi@>F8cz-9p>0|kQ&#$FSTU`#w5*su?K7~rx_CT3Mr>-@G$Le;zYnbO*IHvjsnz%! zPmRw+jq_WuxL*da`DO?xz<)N1x0Pt6k9sP+DuRT_t2Kt-i| zwJZwR*emWm+s8BVt#1#g(Oz03LaEg#{Q~#n)HK|&rO|P!*Xj-|D3E3X2-!jnc!qY5 zBI42fQENmfwHmGWv=OT3#9Gk4Lwq%IeT>mm%|!`W$L|B$>;$bLq10;F+0$mTP(%Nz z)$WmUps$8Ft!M&1Hh)$@DSd;CI@v%sDP$e{2Dam=T1!Hy)v}AHmWf7&q5fJ|7T0=sq)o@2o4Npfq`bTtkJD%&U;jvX!6~;(13|Xr?0&8`P z){0PSwc6QJt7!==${b&Zws8t3R(vjV6c+qDm7& zw$gjPI$=n5p4J)>O07mwPmK~i#oy?y(V${HMHH7$D8Z4B)V1P&hpgRy18VoW){anW zwcE>6yF~r!UT^LA$Xlrx!;p1)tjXD?)Mh)P1IzOfA*~fn_IeMt$M<*h*#V%31I+$z z1Lku*16fct@Xmry?xkt0D0vHVQKJm5s+eJ(@)&Y8Yyt|Kx*PN`awz`u4$ohM8pY3E zbk8Yi-nrjQGxvKjCLE=@gm3~g(h6;^K_mkE#f&krS*wdM`9_f>gHtSAbgYx z62mvp=#N#v@~uWr!Y7c=$;zeV^YBqrjwZ%@q97v<@pp{>zRHAinZx82=d(9{rY1@U zAmh@q;Px{qi@MruL_I=lJoH|)!y`8P@hl3R>K&=51U0@8e|)i3?3p=Pgy^Dd2~$a< z6xut3mH)1*&B1My+$ER)lJ7ZSQrHM+*0Lfn%jeb(=efV($ox~tI?Xv3NIp?Y?>RqX z+G9W!jNS&KZwI6~2O@|0%!@|kN#JV``X>+uB2{ z`>3q{LRgP*CBhrc-wz`E5P#G8;1s_Wf0F?}-nZ0XAJ9nccE`_k@kDUb3MV%K9duuK zip@@XC%A8@N$BS43#H)u(*0dmnEHk~x4pUhLh<;%7y;Z@nEQtM02>+I*VX6}ia1v& zjWk_ocPVX=(nv=R=}i`lx+EdIy#hLgC+FuI6mhodjE_6>x$+qXMVvW6TVnM@SEtq> zW9wpjhVxc!n+MM17)%aO{vx_+L4cq;PrzsW=cLCW&6;(zo3kc(`~kV!HM51ABl z2ws9=&^>mME*z)2Ku8B*{xDE$Uws_vsm(0Py@dzM|MS(Rd_vibi6!*l3oS-KMsRiXq$cSt)=I+hxsF9%`#x~-K{4zE7O`0O08x` zcxskt+WZ%P&B~|RhNBlep{JyJ`szhmd6w3SP-?Zx_tfeV}TVnESw7s?^g<3t7u? z0kwQsYe^`zT7KuLWg;#8+*`}x<4;2crFMct)+&N>yL%sL=~o9?Z=O(UwOZ$?RiX#Z zAA)KXpRbad(cd9!mmN?$TG%7cDuhz2-A|s{)kAipTF}fb<12P(oGZk9oXm1hlrOEA zB^FKTA!|D@ptiefZ3(4T+q4?@lyMzuJEDc!X8UVfK`J`9q`ahxUu%f6V*Qqv9I~#b z`|2vA#uTkBq10--g{QV>p|<`NYA6EkAwEojx3(inrd6C#g85acrT!VRc9#d%?kuex zq10*@X;tmU`fE46Vn)@35~HvqO3a@jYj;;*?dE9h2&GoLtv$7)=P&E-$S%@75TGtkaLaI*IjoTx&%rwOZ}zsa2w9hCljhg+~SZAxhK7 zM!QKNYng-5+dZ~RE#K8z5=y3)G_xmbGWTv(E${Hw^2Fk*(&F)DCHPkCgsGyM9#Y6! z9v4{4&$O0=lBs2Uc}BLUmWle_1KwJWDXuEVx;N8|_OFn2qciQ@El5`7Tdf(P)M~cB zr)G(s)1UCx?6+g2(SAE7WNqjOT7PZ+tFc$SQd7-9NyXJ96me>lHpd{U zaC};P_Uimt{V06Z84JQ?Du>qv7sk4*!Q>LH435%kg0bfBfKQTkp(X0Fgjf>t-vJ-% z?2=&;@~R}{$bY*(*hzOPq>`XGbx}@T%K_78(bz!VNxi0LEcNoJn@z+}Xf{b6A7@k&i z`u(9((&-6Fr_a?+Pe?jFefCN^Jt67z|ItoQsKa)87#H8}3m)Hd?0BxPeQBMr4CVD+ znvBvyljX#Cg$*-9m;9}6(%#%hvk-9=56tw!M^J$x+ed>IS&=sw!LJ0r1LK}wqH(7M z$I1@OD42U-HhTCqs1#!_x_={H_fqYs7Ick-V^xUocLv|5#v6ptF3w9+J%uctRq49l}i4OE5_BQKL!NK)P+Um?0QnB-I9$(ba{ zr}&dEFv)Ryn>&*vd8I%3B9nY(JUNpj`AmQEMovB_tA;|oK_maQg4Pq&0Q#9$P&5`0 z)l#TTOr_h*NCZ-A#MfjOA z4AVjn6H|qWBK%CWJc9(W$iu``VWJ2>Q;uO;>|tW6Fj0h`Dc>+PdYG6hOcdc~Dl|;X zJxoj$CW`Pg6&a=#9ww#=6Gix$Dh<;c9ww#=6Gix$Y7Em#4-->`i6Z<=4Tg#6agAWA zFj0h`X@Ox{<6&Z|Fj0h`X^~-C=V4;1Fj0h`snIYw6l&LCsxVQ6pJ|0*O7}1^RhTHk z&$QAoWq6pFDohmNXIf*JdU%+aDohmNXTl0N*7B%_iK)Uw5q>6GkwjgC*7L*#!Bk<= zg+4Az8Z}Hg9ww#=6Giy%lw+83Jxoj$CW`Pg`i6Z<=g@!5SVPdKN7xOcf@I z@G;F=wg5y~1}n3Tcn~o2ju$JsSJ69)wg8`S^igz*qPdD5Q}nr_olsKY9j0iuqQ#2- zt!Pg?vk1o+MROG`QS={0`{PMWIEoZqt>`&L>lO9uVmV3`-KgkQMd^5ilAMoFRHf*4 zMeiue#B8>33{rHCqD6{6QM7ZqmhphV09RFH1PX+dgFpXL#GEEa41q`;;Y|;ccI?gfqPDQC=JiE{+`E#pzD_QuGRPrrJj?9mwwX=|Fb3PY1HQeL9fc?bCtm zZl4Zhcl&f8yW6J&+1~6mfWOw^@AiLWyoZUSDPfxTvKo)}$!eUL(@_G~P zHG?9~$(HQiZd?x|u0H!q>ghgn0TlQ2R zN>^Y|#CZ+qUrc+$pomk1cVwS4ZK6RDXEo4|Orw1xgd)yNypT#~8JufS#7V=ut{s?` zZcxN&0Lo(8RR%?zT_A=WrbP{kI17OeXWCr`MVx+J9p_l4gv2>P{f(Iwc|X-v?&HfoHamCG3{G}BF?$k4)ZM2Y7B}v>DcnI zf@xbBgoT1Y|64DSZ z6q*b-PlAmSmM5A7k7vzi$ou3caUWT%Xk-xbK1E5~M-~^h=03BQ)Sz|Tx7rhbCv2iw zOB#}6TaX;vqU6{blVe+v9NWs|*w!S+=J>i+$ac#}jxCxTTTXIp`N^>rCdWojLlQlp zGC8)Ix zubSt-Htj;`l7S(HHy_Q-S_ql&u;G$kl%hB7LWy;$G=g};qee`XCEOI@mtKuwTIpe8 zsxVQ6pQ*tx5k0Q3Ocf@I@G~tiOyr@*nV2d}>bkoccadRQ=V4;1Fj0j6PK}1ip)lT` z_%H5H6yaxDVVKfAOiYzKQG}mqrD4kOFfmn_D8kRQ#xV8pFfmn_D8kR=V3Z*(k9wGx zDohmNXUZ^4*&ZgQ3KK>6nWBa%$HT-_VWJ2>Q;uQE^)NA2m?*-}ly8{wJxoj$CW`Pg z6&j|Phl#1eL=k?bBEwYZVPdKx6H|qWBK%B^hN;HG#8hFT2tU&b!&K{GVyZAv zgr8}pVQTO&F;$o-!pG#^g(62?&@L1~7c06~5$!^eSlWdmh<2d}qFpG0Xcvkg+Jz#B zcA*HOT_}QR7m6U-g(8S{p$MW~D1vAgiXhsBB8cY31<@`PL9`1+5bZ({M7vM~(JmB0 zvwEgnLQZK4l5FIHY~b5^ixwvt{W8eI?!9C$6`IlGELAi$mI{Ea4W1a9J)yg*WX& zVFggbyB2?Lc)5f#yzNt#aE7;i$`a1-woh5Y8Q%6OOE|;ZK4l4Kc-yBe;S6v4lqH_QpemuU3Dl#uK~`2p{Fx;nHAg^=t*>B_6;X%`A1Y_Px^v}H`AT_}WP7s?Tb z5-;sSAtbv{u4fwULLnr(P~K-6?Lr|WyHK{uC0=^JMo4y{^kN$ALLnr(P?q+gbhHbF zknBQvgK4x2g^=t*`GRS*3x$yELivelv_VXrMRs*)7YZTSg|Z{lXcr10*@dz< z(`Xk8A=!n}n`u!9gplk)>CZIUg+fSnp$uXg?Lr|WyHIXnInyo_Lb40xZl=*L6hg8K zAe$|p>t zT_}WP7s}^Mqg^P3WEaX;Oru>Wgk%@Ww@jm5D1>Ae$`4GVT_}WP7s@Y8qg^P3WEaY2 zth=-eg^=t*q0chNE)+tt3*}~}(JmB1vJ2&QrqM1GLb40xE~e2g6hg8KX|Ikv{+*j6ORwlX=k zHOa9#7}k=IWJYpq(d5{2l4HwHj;$~`HgX!0kYr_YY&FTTH6+KjAUU>0$+0yi$F?Fl zww1}Ttx1l}fyw+-#ZSu!phW{{IRUi%09s)Httfz289=KEpfv>0 z76i~11<)D;Xe$C}D+6e20%#7@vWffxXwd*#P5>=GfL0hlD+-`h2GD8(Xbl0h1p%~0 z0kp;d+KK?$%BD2_QW%~g%~!$pR)jeSLmQ%qmN?7T!=@>^K+*3N-KyvjMK3D)K+$)K zwnQsR&bun=t>{=qCn=hyXqKX@75zog6N+9|^ogP$6m1RJND1~(bg-f!icV2aa| z!hq>5MW4Ke5+NUhx)^D^=S;Dc|(LQAf_a3o*$`WqfT(f2A ziH=P6h3t&$O0G|bTffmhWeN8lk;`%+D(+vf+@^9#@!0&fElW7VyB0Fp@Nx-fc-yBe z;S6v4lqH#bfi^RxTaL-%at@{I-=#2l97QT(I1xa!K*n{I)Gi2l97Q zJT|{=<75yCzv+MAT0g` zTE?`M24V3pmMK5av@(OR_!sCErmZswi+{09`Aw$HF$jx)G2#6_(=rUg;$NUon0CEE zSp1vmIIr`9;B14i_!sCMrafQ~7XNm4oDZ0mZx9y$0)5J~7YxGU-)*tHoM~eX!s1_` zHB9^1AT0jf&T+nDTBSh|XC2UwO#9iOh%;w<$63#`T7x1^#tx2?#=5(mK@sPApc|RC zz@Uhey(2ztz_bGliZ~AdEo9nagCb5omL%WJw1EaioEL!ZW7-OXBFI&8KSF@$S!{k!{8}W7>4}T-odjxIm{i3T6>4e z)-g0a8$ZF({g7pP4$R7@ zgW98nVtoE~K~|L8sX-!X=`a#xO9Eaig@(~|hpc{!E?VY*_w9xfl0 z=ioCA?^Hddbh@*xlbIO<>3JqKIn?t!M4jXe@{}*(8aZ2J9Bd@`1}3m*Xeol-dWE3yvteVoX|o( zDd)!>jyzyW2iiuIj1=m`piVl9ye&Res{Y%g^x^MgWY>SUufk>hRw_Kal507I-}diP z_;=pe3LkG2PD&JhvQapx%nEOAsLk>rz~i}vkD-mIr@|UayY406xUP%VA@YT(tT>N4 z2LH*C+I4gK4x~}n1rWCDT2ooyC~o==qcu`>-owc8DN1W!_Nh`@3$+PtHx0Db++Z$( zYU=SoX_;z;Tnf$^PAq*3cvJ`)@!lVapBz)>_O>lvF}Jta-nne@gkz68dD5g0M0hHg z=JYK{1C8XGuCE`1$jK*myv+nt(a0ac%;D4r^T1jq{;7PBzqjZ0Gp=ntOvg1+=oc}( zTYSm~jF#&cOM_Sn-!GY(2AgVG=rrXAcgkN=v&oc5jrkLBNl!gXryM?Wsa3aZdr8W! zLGW>tij3xSha+MyahO=?% zgL$t-1%O&)T;}l@)IfP2M~Xfe?;A*|(Yb1f1~=&fdvKXyAK(douIx~g775jJF{y^A-A}zv!~ZqT!@=eWk&hKpRT}@Y;G)a*Rp;J1S z%r3SsYwrvcnS5DF^&S=ts82@KE36pxZtsx#B#;!flESh#z*2ZG`&4=VY#(YI$uI9% zFh*T;->8d@QrELo-S&f$PI?;Ne}vLA&G4j<#=ij0d`IIKfJZ0o#`)i_lbR~wRJFKK zpkw!klv-UsCgQYrN3|-f7_GJU$XxQ+$oDM;zGXCS51^6NZ0?R$~ZX(_*U z+T(zlXqS{!>xH)kR|&7Y)1p-#uiDt?n-d34j=q??>PhsSt|%RSA3?U>x*8d@;V#lFv;M9~Jj4D=GucKm5Gm0f8UMsvUP^|EbVwH2c zEj8A#g7Wbu+_lhgD!Zk2*ipDZ4%JYLR!|Q=dIG^u_*H2jryql~*3fqAZR5U11JZh9 zwSHgxCx%SQDsN(CV^t%N?P}cbmIStJXG}2puyXHL(V}8mKctkXr6D8{cg9jJ{D3AC zaI)?C{m6lZ6Q%#Uk)Bjk(rb3*OY>`M^G}YF*GnDB_T}C2!>kX|HpYSa>E3iJn7_A2 zNWyEuS>Q-W6QP87wU>)ZxNuHkX-8pkZ+lngO!OVVR` zF?Ywd6LfhfJK1cIB86oUajoz+_*4|V!H5rC4`2cxT7^5OIGETzG!v24Kc*GB5$p?{ zi`xr50K)z=L3-Ou&OpbdrF}Nbg2%%i@pmn1$oS_WTBrm0>!*HJBc)GYXnODJMRU8p&RVzX3uG*F?|8tP-9B%s}GcwaUYD*OH& zHK1O_8K>OufltGF8fUkw+}Yjqcv`tA*Z(`Ea=XUFDEBwUU`a)lJ1~S&+{aBp#i$*w zpIiXWDUOZq4<5yME6%Hf`iW#4C3$Zkg!(I`2(@@##6I6+?2}ZS2$t%@4o6``i_Gwq zKp+S~z2KbWh=u1Xu?|G~imRQF-|>%>LVY#tbhN+$BNC~wNMQ-;aWyp; zX4SPUirpeCid(XH-u`Urd2 zQ97m+V6U(M%>Fk__DO}aA63JhxF%4;2XOysj``S;`;eeipV8N;it+`;w(jmiky#IO zQc7^ATZ&VL@0!w?R9FV3g4(UP%C5~0gv=?gB367p}C z!+XJ)L*5-%66aA1u}fme*w09Vs=6wPA*8C6Tn;Iu)FLG%vGCU6YT?%a=G65IJ!5S;wEnnz>@}2q{Y$5UY1Z3@U{gt2>Grk?bytU%A z>v!k*Nal*-NFLzGtC? zsWfMCRc`LN-T_sY9QFSPrSMY+0=23S%XlE)r)Y58-Vebda#Eq$M2^@itQm z80YGgI6WQ4DP0R;#R&1mdV?q_=@Aj$79b!z+ZTsxLG8)7%_*GU*Iq0vDPckS=ak4C zjPPPbEn3k2MqW~tThI;0g6@QaRAND`>5y=2q6{$3)hThB9L6b~1!1nJislC!L`jJS z36B;eyet-U7w+nDJkO3nA8~M-Rp@PFeWH|8X~fE1mlCU9R3$8%l6Kf)g8`|^t?EHz zRSQt%iLvn-LwyVA6u++?Eo%wPEa7hsfd72jy^kGUo3fR@QD1CWUW% zaINrO^rFL1E4wo#jyfWh15RI;5~*K?krFFgZ7?7emN!f3x5-%9Pb##sTQj2L zm`hr)oZC`j_xCV%X{-#ZOsZJ+L4zDToTf$hSv7!b)7R@z?k`gNt9<4}tS*)mBiTC{&+*;atm#3*$C0&;i zsqyF~h{UEZ84O5OZdJc9R@H%KbevU{t5-dh5go@|Z0kw_%egHjcGJVyrLnS8MoNjd z$skNhN<87c2rCm_HLdJdDRI;>sT^?nx|B%W7Dh^9*Ruu#Qk7fTAB>fquBPXr1aZa_ z%URifWkkm@m$YCxx244H!7z4dtZcxyvL70RNr{ySk5(qUELK*oUiHONI675>2w=|d zDUq3jo`p#4y3WWEBZ#vnh;>#HKrp%%m!}qUVhj{iQ-H`btVP9J43~ zj&pfRtbP#2DvcHG=EovV9hE+0Qes8IqZJ7+ixsWGUGH>SHFg{CLhDQfzA%8aVk*eH6E;bf& z=h(_DWJgcL6DSKi1273!V4^9D|mVocW02Ff)au; zHx%ZuauP*=MG<$M=tLAdmhKW@ z!ExCs9qnzLtzzf|4D2!2_2KyhQqukzIT2Y%h4ancRcre!J#Q(Pe}ifK_Uyu7@(SXR zKXYONsIau%;3UnT%sJ8)mlGEAiGC55NUZOCBNM4?GJOCtCCz8KGDaqu?q$m&oRl*BGfsvzMg~&Z zWaxwp$0vCHb72{#HqY_%4k~4yfAaQ_CkXB@m^_opmgipN*`0=F+tN6?yd3m}sdF)r zyjy2m{gE=~n+^!&92xaaKx15@6{&3bj*B~XzK?YBJ%8F1ookRO^Ze6jo~M~SlggIo zawWrO9s6ioyd2*ks*+(U>6E#C#4R0(H5+DvHK}a*F6Wz@WAfe7(TA@k?c56|r3{l#7|aaAfFO+1=jJ#`&<*+Irg-Vd}q@CHhNBnf?|h)1Qn?q_WAh3VLhr!_bGHUlbUj@O} zmj7ZI`+gvt|I;w1ez~7@MW@X5m)sfC#C_~LbRCV9&iBH{YVg~uIOFoh!g@#lY#0wh z4nlcav8zv4)fKJsrmtWcE$QpQJs$ss%Ah}jnZv0M-nJZ>vS)Me?)RqfBNwS>lB)C8 z;9B8b?NeGVtfADxPKeh0V1FY-PzwX(2QF^NdA$em2lPyM`N1Z&Ud#AL`7*Vb%WLz{ zk%84ACT`m)|0ELd@q;{Py#Jg=KBv*^J&-@(Q!QMep5J61(}>43@iBRMOg?zbD6T+m zzDiP09vDJagU`EK%^w4gw=8^Y1MdB(>H~TNlzd_L1gOcOej)ZIxh<_N;IU8YTR0C3 z`Vpx;(?J&eq4d{+OZ7oT$e~N$R{Z((Mx(#oyc5Qp>|b&%*!=r)|L-s3Z_d`Ib`^*| zpLya8Mp_-MlE;srUFP%0NarB^3DS=u#SBK(hxS8bxumf2IDQk}Wj-bSYQiHDa@_`V z-93gE^f^-9c8?V7(&@?R0O#0*+Rd{QaDg0zapqFrRc9_u>8G2*UpNzwGmNaHENjk{ z&u_V!%x|3hb}sJdTT)=}lc~1Pw0S9|_;Xka(d!Z;2dQjw#C^46w>l{Yr;CPkTA{O0RDD#wNntE4%ugxJ`C)0Kd}ufFkjf^{MUclG546jNGsE)CDXi?e zpfIgnw-n9aDf53_T>dXI`6pG~{J-1He@|Cmaj{w;g7!A2;QyU6|M$e@|8kRmQrYs~ z4gB3vIUD~);ruhtz6p^`nd>LQxn@kd!Q`1#wmg^X&FpmZysDRAb$E4lW$Vl@?EYz`UxlBsD|oq$8`vu zZ}U>hF&+h0M2?>sIY^b0gKf%S%)Oi(_k`s*tF74Hwy2|^)_n5y5aOkjm{_X`n z@b6xbGQS^)=J)?heo2M%%lQtY>PLo~@?ddNh-ts0b@v{`lGDwD62KfYKgL%qO9>zRxZk_kFvZd>lIkGgT#wb)|M!1D9YLx zWv!30)fmx~p*5i$4uQA_+OS8tU_g zdfZSyG}Lbm^)EwFh4PdG40W8LW*X`}hUzlZrH1;jp*9%mONM&JP(L@+pAGdUc$nni zErvS6Q1oT_b*`b78fvwnt~S(dhPvNS-!RnkhT3MRe;R66#D96)zJ@x|P)&w@I{K$m^;SKxG4c3+LF818$JHk1taTv9I@^2tpZ_Ro(C`z(y4|D9Ac?5n+Tf}ZTa zw9V>Uf=vVRFO-(_QgGZ+SkyRYVGb>`NNpI~$oqOEMG1o~a!_d*8;lBJ0<$;4={taB z;BJ|Fdob?zk1nhL)P|AqTxa1zl{x(b(^~UKu4zpDUf9c_9-ISk&K|LF;nJ?+%C-)# z9_+DTjl>Yi>=E6)*7goLu=TCI8fiY3^j|{W1`ctnV3zp@52TghSU=#%TYxcs$ zNMMuxhHs-E8oGR_cf?a$$vGNy-)0(ItNNSNa-n3i7xTl37^dtw)*pUEEx`N$i-p++ ze}c-d9ic7+H@7ywi$&M^4^U&27_L-fmbePL$ZX~qC89KP?~h@^5y*i&jXkH*Ga4VM z)l%?cRA1D8&2*WaxPq*Dp1l!1HoD+6Rxb`aY76et2f^?U#4#?>-ikl1ZBTXxYdKiQ!=`4B=;~dLIdF_?bl%^q zHfB$8vD%A`!n1RI_6RHi+tSs8oq}Z3n;)u>t!|y$eo;Z)-e_)Lf-bb(ymQC!D>cs> z<^2wi&?BuI6;J-7yjG4QkKspiWHl&k~j>h{>MEW+QlaU^V^dzK5 zAUy>s^?Id=^L+R@1j^Mbn{9XqatR(0&y%NvPMDaY@SFcjioNMTIa8~C6;fRLi2 z$J=qaf3t}m3_fxtqX+8}gmtXGcY5a%gpeIoSfS($D`z4PLH&X*rw`)5z_KqG|MZzN z;K;-VKBoc;KJQn7WfoHAcn;EDq-cWmUx}29R&k2QSx7&RbOF+@AUy}^w~)RE=`WGK z7wIcV&qd02wIbaQ?{7oOK(YwwiAWbCJqhU&q-=67M>-d2JJR_`S0Y`Cvr59Qy$9)qNFPSJ3hCoW--ncXzZfZd>V5{hk$^?2&ij<%F-!_?2V5`L z5mVufGrXMP{ijzbRv^~ZGYB;1M{dp!s-5Xn*g}v{VacvjsIb)-6ShZ8@C!U>;a2fQ+WqjB2og>2?DXDcPG+c(>qc;bnp(+-fcd$@uL9 zNu;fu29jfaNIq>KAtjIqkB|s26C~l5aydxY^W-1Zw30~sI1MC-XUa-GWgsCXkO+^E z2rmLj(5+Icmjx9*Z^y&vLJgpHQdAA7S8=Yv=}v3q>><`Qj==fO0496hS$2=Iw$WfM zj2<-$Ehimmbpxvje0?w<+*)H4QXTT2160f#Vpl)M_I`kr)iiXW_cOoGBV}BwgOpN# zAXVoPK;foJX*qTB=#>67}We&iVsrqHU!Db4uMstN-9PA<);7&;g;{osJ zCzS61b(ErL08-Ux0@V`&dL20KSKw{n5$JPp{!m*CKC)Al1Wq`#_GHyP5a+x%WnZDw z1wjL*(AYc|`^&6SRe9IfnI9J9c8P#{l!3Mw10^LrVZz&hyM$K`bh>j&&{2n+*ht97 zv)x+};y2=L?zd>?f=9&9#`#?raSj#On;2XDKB-Zk9ITU|!$vvOnI{)@Af-^>Vo)bl z=Mih+-Q!bkSU}aJa#JVjs^(rf>VwEvlT+KVQU9rndbBc}8f>Sf9B1NcTiP7*q>nhX zwT=JK1Qt?t-g?|5yxV=MJlYv;eC+US{FBJnSqW%A?xL;ce=)R_*ljj*m^t~NP;8IY zL3BtWP7B)RH={t2qL7qqBrm)#Lt5dLi!>}+Q$`vc-bcdrgFw^&0Ug>06qOX(FT=~+ zGW5ORG0?Dm$&F+EBT|N@wsot9pEj^C4{jL>U{qjPZ%mz(l%c|787jO8$foM1OVNH@ zn+oQ~{8F@*`SJ2AP?V0%kK819v%dGg$VH9ECdpL_Z$fsBqK*Y6g(~X=QmD=cwF?4b zP>)#)&dH7=EC-J|ScLQRKnL;l!qke0sTjHu9`9q(N?*_i^TN7e4N}(kc%Yx5pZV*@ zzh6a=VmkWKdR(o_Ouh&dE9a_uxF`OUDI~0rcZUFjM@1%+oCMx(IN5RkQET{bt{O_8 zG^w>`aa%`QF}H+Um|}1p1Uggt_^JQFiB<6H<9cPJ=bT=>#Q;I7&PyY-@?lnn+gqzMu z5c=&=p`FF>s)9je3sP$STLuJD=|I?m&V-U6a0kmc5NIy$w|lncF9_UUmasDWY*xYO zon1F6_zlRXXrEL%!E?B;q~J1*9UwY>emOX2CTRZ;uHf=1N)@Dbwmn2XDEMVYa8l_6 zZ^V5i1?SFvaT*`QP4f~2|8UR_a9g=mv3F;?K_&bXMtD-`glBPFQh09p7bpBxxXJyT z%{KSOz~}YnBxZBMDyTkoqr&eQUniAL_~l8$%RbUkljmSnK0)&}EZ^jiW?vN~-mtrA z^Qm%;rhcr^HmP(1Cq{T~$`_~YoAEMt!#!KuUz*^Hi%@`eEV#@7_dv4J@@U41L7mNsWn0KX9s3aAWCmaQ3V{FjR%$g?}B_w)F+h=h(wF1F98Hc z*xf~yZHt(j4v6hh4~T3uNDJA+07EJrFo_nje+gj9S;+EaVDz%dRVg73MrGHYc1$1+ zFhGz>2SlQUaO=D{S1D&9i_-zIJuD=$ksyIN)Br;&9WaR&!aerlfLV{1wI{@on;c-w zZdxU`F|9%{arO%&AO!;msdRuOS_(J9iv#37ylhnhAlGLEWP6y*S`@IZD|HxvNTmZP z(PFq?UK~K>Eao#Q0jg?n`DGXpTFmLYfRPq|yPCXd&F^E)JM-7II}7V5%BRoW1&JA+rq-q|yPAXd&E+ zE)Ixt7IJAyAhw5vIQ#M`3pvXGLn<9Gi59{=>*9b}4=p6sQ=W?fCTY{Ss#Iy6{r6}o zUon7?N(V@yrEDqz$bEQOQa$B683EZI7Bd}#jcz^VaRU&kbO0q<4ELX_#A04b2~bsI zi?f#=E#`X$5K`#?NwgSl4HpNg^Yk~4~ud3%TqDt#|9u$=>ST!815|>2T-|E z=%G>oZI612vp*g!<$41IsdPXjS_-#+ivyxuDfIc2Kx_{SarVMf7IL!zhEzIW5-o)L z#>D|s&O*LX2AHbG5@+u_TF9LS2vX^QNVE`c?G^__ISUy|3B>lW5NDq|Wg+((U`V9{ zCecE;Ph1=@*Fg(sCsgq#Ilyd>(BW)wrvTsrWYQN)Djficmhn;v0B!{U%Mt)MHW`5J zVGxmx?dU4MHh_>y2S}nta9_AeEMjInAXN<&&W3i%BK~RsAe9b)M2p~Rad7~YvxsTQ z0BjG7h-_p>i};rTgj6~}a=5SLxDEG2iv#2`+~lrCYY$iKd=xH_gM(N8j%`)w`*8Mf zqtJgr9wjc2N+)y^?kg!Yw?~T;y4+-x{T!jy#%)!Q+1blYnf^Y=qX_}BX@M$FkFBjq8 z;Rvp_$g6_b&aQbL)a$Y5NrWbqPUwlauVk^t##WrR%a!FD921v4_NpMYv+Ev({yP2> zp-H6^I+r9gx4nxKx?E6uGDc{nCbL{4XpD;p`s)Eqm zA23enLFh4QM(y#A(4n0St029zsiD&NTq8WGDhSU_1>=Oj8p699Uf504>5lOBc867Q za%VF{L^H3|2v4dC!gDvmIN{3`Z)du~(tCp5k!_D)ZzpKl3kni~~XL1Jh3Lgax$Hycfps)Eo< zN(g;5UY1m8+3E;w?`&8V`JH_aDSW{QPpS&Sa|6OSjbDM6xeLSFHMYNpg*RIyRzdBt zTPAiI!AVs?aPD9jCwRI3w%1(2Wq-vg$n9*n2nA5%?=ymvs)FF$?l4a9azo8^!MGX& zo4XTZRV?1wr%?fLr2&9c6#(EqiE#j21GL?>Pi&LlGYkN~fwFgXtbzqN+dKLe@Q49` zR22Z=n$U3olr#NpVF2`YkW~=e*&osu{8i*riZ@bK5PSs6<~YI2)#CmqPH@>zvI@dG zn@jq_?|{6D@T96BJhyd>6aIR<@ZAYzdtLCUFa18c=VWy}AEs^3?UdlRzX5?%6+qx# zkZ~Za0|@U*0O1f91ZStps$k&kW69`$o&kVV6#(F7l5qfVE=tlT{62af)3U2nwtb*%1J7-exD~#Zzs-W>3N(f%Az1|iUoW5TbWjp(B zsxtmYBQ&Wh2z_e_q00?4taOFeyL47T*_*?Tw8rrSa5%f z&nigmZ2C!|ziNagRRy7cRzm2j@v@{A+%1mK_FkY>v2|ysPzwKo5uQ{!;p=f<$u|3M zD-{0ju<)S`M5`kF>8P|>_DwU=uiNx2_Pa7Mt4%pxvgHr`!!x zKYDX(JT7DTqMGx|Wwy^(yc$ri;^Znl=Xf*5ATdV+s2y?ci<50MRp5XK-x1I?BWCN$bB1G#ISbKo_umXk<{X&7Y zAE+=`9F9BJ0qYR(2-X2OU(OEJoWkO+;*!D=0WBR~k>w4U2N$rhK43pLfRU2r4TZ<$ z4TYE1uug}y)Tp%ypF|Z@4C@(1WjCw?g07I#hP7;=UI5UUZJ!RPIKvwIrm$gQ>0V`6 zPeTrgG^E6^gh#^?URsCb|;8+P~m*iA6lkrM2LN9=@` z7Q4?I?3Nqs`buEOXt^rE?mm zhOE0wnJ`!(hM%eehQl$$G#Gx|$M7nHAt}L7c*Ib6Y3=Z12E)k)!AU9|>_`cA z!XtLVON-qL2D>4H-P0woD;NK^WW+9M5tQv^e^W*ZhEEv`Nr~?YkG?Csv>3i@F#LhR z@Rt>0_+k|>+}?J0xsTy542GlxL*Wra;ibiJv%&C2gW)GjU|25x-CqfY|EBo&bKi&8 z8|+Al4-1bzEWEVXecxbrm%;9f6=Jt(7}#xZyQ}x@?jeI6DZx&7#7=l=vFk9{{nE74 zUMbN|8^qI>C6v3zWx%cq<`(G#U|Y>?+wS6*w|_8gHBzz{gz#?1UBXL?*=mE?KMZC& zmaw;Sm`%@)*>KD(6=sut%tjeIBPEy#kC+KB2D6w>xJyDEZgK5>^wp?7q}`EnJi|Xs zd%q1`Y3;q%J`Yvtc9Xt!b__~elvHfJ-^e6=WxFAJvY$l_D0U)p%zugF8@5cpt5l{6 zddFBtV*j1UL@HUPpifJ7ZqPE7(^Er??UvBfFnh5?PwZzCnMfte6!bsITIE`%L15#? zFk4SwiRo)9DN_}CoTw}Ii-}yMlI05eiDboXEf@Fa?n|)0`B`*T^`0ToAN#XJPEyHo z27NlRZgg5X3&WLjSkYDFWZ#p>Nh(>+p#McyJ=Su53i|v&fwS%1=C;IwRI;2we}Sz2nbv-vtw_#l_YjD`vVTM5C6z31 z(6*PgE7QvR^Qy@^eEqU7LgXctEN{>bmlYVbyyg7$fy(8rn!mEYK;$HqEN9SWmURTv z%K7zS%Q<{~vTZMNl1i2{Xy3}Jcv?=*)Lxkoe~-^BXI0dw;x60YA|I(_`GU5ntnrps zz8RIuH$3$z`Pfz#`AFIOWS+Gkf62?Tl;7128=#`CIKBMZY1}0J;)a|z`igg8BsM>I zb;Ek|D>jGC@e*uFZL_ftYU2gFb0c==hr!F#qw|CQhNrj{`#xT#o@g_dho84%NbLu_ zJU?=CesB|CI5CgCA)Am7Z`-ci?64{S+7S6o{3FNh%$s_ake*0xw@?FkjJg*Y>wbhy z4iynn^T++6+@|B7t`LGHalPr>0MkssV? z?TGv(j}y{`=RP(LiEQRN$x!bE^-tA2KM&N-Xe`!v7lD#3T(2|KCqc=*91FiT)Sy2R zoa3Dt|Lx%IfqNDrZ3!B`;SFA=ELozqYh?cPQLhOvhuJY>uqXX`aIBqO$=U$cEIfYp zh$V&Y9&aD8a$x=NEo61}t;B13=5_J$dwC~ToK4#2B{9KfG;8TsR9>t_2!t5z7CewaC#driKj@|HpCZwDbgKgchrTdQR z+|4_8%xmDG|HNB>hnv|y{fa675j$>3yDR?e_4!}Iqyt>#~U84LGEoaym^Kf-f$Jx-rUf-6;D=P4nH)1a!bwpiJSX&Ir_Uc zcUnWM?W7awAK<|7^Yl>de(*u;V;uM@{6${}C*@njH^9yFrW^xBF6@tuZetG0qGW`4!WP{lTll z`Bt32CO~q`E204E#9V5WT~pgSIu^ApULg}B6ZbV9Q@tKmBq*>A?5HQp(pMk}fr6t~g|JeNhcyN~Bt;L&qiaLx$~I76p-oR;Xj`l!`} zDbbt2vG%K^mU3Y2k1_zxu8g`7e~Xb@Mr}Wp`KD2iM5>$?=?V<>Az*kNNGXq=p1C{F zl3F0CI@T6F;cf9LOIJY0nHsGeCk7bUBw zL|ZmaJxzF;KBDR&bk=5(T0QHssE7G>)w9u8&*za+9z6r}5ThPaq8{Nu1~484DM+I9CgnoywuueQEROk zb)K|ZWBy&Oebm?5$BrJlQciqS7A ziN35g^^E4njGjqJ)D<41uJDYWQ+nkp+!-_`lD#r$V7Qv3SBn0! zmz6(DskN$lr?u6e_Rd-~*@okt1HRTSL&~yJ&!PU2(Hbeyn((MK;Tf%^^iCa1rCwT{ zvhrEuopEJldGCA}(j=9Y8%ybVdz6*dMtxepU&>GyimSoC;78vVkNR~Oy{#Gf2)8!w1IOLg?=0}Bs}`KcIJ(O2 zP~-6webj0q)HP#mW2aR#^^np5Q;n(EPi1~x9o6}@&z+G{4n33ozY&S0Ylc*v$Idh1 zZT2avO;}OH%$Mq#DXpMr*G!yR<|2bs3%fF=#;YZR=QQCd`iQE9kXd`lX|>#+MJ>#) ztCmT=S}@9~x@7cZ`F|RanFd=hu-K2iB;v${k2KRG5dc2D+e z9tvulqTt_NFaE4igLplPY6EXS^=l_6_Fe@&q4$C_(;3591>R_!MVz}IkN@PDQY^Nu zp3HSoX3m~8Nu`}jyK;w~j5=gTN7rJ^r&j&=wg)iN{^5`MH{s9A4l>opvFP&5gGE`? z5dZX!!pcHtZ;!{3ryRI4l#fL@mUW^ZdNz9fS)FXcsSoD27D%!0_!ysgD+(;i0Q~!B zAYG4iHc}wkPq{vY6lJ3~(&(3z=vR1~{a@*J_MqgySHHU${qCBm-`z^+cLSuDnxNl$ zSHJA<&Mofh?CGsUvs1$}mgl5k3@j>9{i@P>S<`g&afzpYQ>v^&6v#c694 z73Z|)BAYv^Usr$%JME{zIoVNaA9ytQE}ZuthyUadr=23DV|Gvdc>OT+8F5tRF^i*8 z7VW6NMA#CAl9F*b;Ze21tG1)IL!oQ%^fWr%U>x=CL`VHx2}gYx@0si9)DFaNj&sz7 zO1-m}ih8j?QFm{9S7%#Cc18b56`;M#+k3q1A-S54yAPZ}^h`?Xs>0g>goO8c>iJtn z&rc=l`I!p!yiXcE>*`c?Juiau_@f-Jub#Q?qv)BG=vjEwv+!O|Js*!co5bizib5u1 zD^`=$uhELtCS3oX1b^NyR?p=|y0R<$J`{fyD;zRZ)3=L8=cMYq=ka{u{g+QwzfjH+ z!)Fn}3ce*XI`}ecM3FuYU^$eqHQgv#EqVS%>n}qjz>Uxy%@7hHF-noQ- zm(%szQt7&KzixU1#q#$UU6T@B3y-=M-s`FB&lz3co2cuJC3Ib`{64s(u9HUJvitXn zV5lv7g{{i|{Vk(&QWDFB$5<}B*HhB4(Gb^T$Z>zfmG z{qYKQePAkGSMJyUd;{tFL8EI@qHEz%*TQ=}b$#ldy8S&}QOIO$`#Z0Gjkdox;`%cZ z+TVM`>U!Ajyw@A0AMJN$V1twlYzU778^U`%^}E{W_u@qT_E)Iiy^{1>xeH(RM$zv` zqq14;(e}i)n+^# zx|%deksO_z(z#@Iv3*&4XZA|>$-^6YBHIB{K~6YpHYi9ZJs@{VF{C*I9uq`pSN4g6auVYWwPfsj8eBg5PTu2Vc79uc=W> zR^;R}_gXx6D^B&%Jw9_+i)S=*QHh=MYjKm@EFa1@t)0TJI`J7DI+eT~P)Df2_@n+3 zxdzm$I3K`ynxn(Zz+)iBSFzqy*E!>-hje!rI%P=+AYl$>*FUVuA>Bdj<&qG2T=6tY zGkj>=TUd#WPh?iew-u`!D>d-*_BrY?qf{4!S$d&GKO8CZ%m9q<3JS- z53TeOD^Bgx^XSfI>52rG{sjgG zY&9>#Gj3A68c=lZ4LBz|3LgM(G|u zj;F3U-BcEmI1kieil=qG5u9m`uCD-(y6(rh)z!5rjAa)%7R6a~z83EZ=BAV-z+*m1J^T%v(8DK_aoIhp9pkB=cMYqd+?j^h?MXiH@wY; zSEZ6y_M*S@l-{E49MrQQ62 zsLDSdQn@a;H{)Ij>D$xMn}Cogom8E7FEA1w(Gnh2Ej*Uo!V4>11!jCRjfIRvcWR*! zEK%j%igF^DTH6oKv5w0%fHxZFzBn&M9{gj~s(f3|@}nnB!rCBKtD!Aots~-^yXf?`FH7WRvO$3*H2d( zL?4egQ+=yI?B1oQV?liY#Y2sEDyVA|H3!u9)g4Pg$sH>}y+_?q1SNOiPydeVKs~4K z_%tZF<1SDO)gAYNk~>)3$UC$@Uk=WRjy2IjsktX`9&t4OlY`q8<_jHITLlKSys&r$ zJ-Uod(PJs>(~Y6HO^Z6ao~)&OhE?lol6R^9nGa5=wF_SsC=NtPHADRtBvoEBhN#TG@|{m5~xF6W)6NS6Z1_{s`4- z#4>Pd8rW*isV4U>4Z69ln$orT6KWcIcFPaqnt`EOUc2qr>gp$(5Mg1Dp*53k1IJw~ ziWRmxoGmyuPF8p{=Okg*%!_zdRFlT9THm~*%=8qLFUQQq8fkg)UnP)6A*uOW@Q;{N|c~g-dhja!~M#$5U zPD6S+Qp$G*Qfgu@E*5_$B?}P>?_vK}OA}a027MMv6G|g~?}y(~$_48HHuAzouFaY2 z87C_X=CB<(gGJ@=+qUP`0|N!2`{OiRwdGK|;QkmmZm4Bgr`T*oZKRxXxYO$c zEdg(HM77j}->hbl?$!9!k+zl^aglIBT2hVPR{W+9=8bCF3F$naX{+hY;89KVU8?C6 zq|=d}ft2cc7gDP09HdkiWz_2WF^~~;k&?}jg!dq>5}xlk&_I2DV4~LxpCCEN#VTxOmXuM?ayY=7`}#JL~NSpjHRxDY}YIu~PgFqC4(_;Hmv0$BATVfFaUS|O1}O`B#Z zrVqZ4tIe@or9JYv8Bv0Hfa3eOmJ&U?Y5-7dg+a1gv_^mQ)grveL#ZK&e2o=_~{`z^*C6=Rxkxjt$& z;f?oWOba*KF$`>Qk!aDBpq@Wb`$PF+#F5gd2mn`pA4hzq?TH+sc(*-k59slnizz8r4U zN3AAMpvg+UeEem|iQnL7G^5M?kBVm9y;8ckj>wxBwQNV+fUokH&(9*wA-x+Z^ZYra z)K-nr7Ae^xOL+JAzfueK*Lo<%Gb4B{o;BSGQ#OKMb`f-n>de?#U(IRk{v5(i1}5w< zRtb9Nm@r04`niO+0TK&u_~@~c?^bkd^d4~03wjkZ1804nrh)sl1Dq-?hl|`xFhVH> zNy(O3!n+;P3UBzxp}6tpC)nNCxGG}8Ct#W3v_1mUaP9)tp@+`*;p*7oTc9FAlay2e zh4(NJ5Z?Aj^DSx6v?hvVM$`J#N~5{!c0ltegC;3KQ+PyEc*95YPCV<}1l!ETqM71z zrp&NfA2exL>%*{CqonCEsFITJhlMu;B!sv9QGI(RREJ}%)+a+6RrUv2u3rsQ*BVqw z397;)s=^ySs%&bXm0+vml2MJ$0nBWp)<-!S&E2Yx=4l2^QsVi-qvs26`=hyUDl~^< zo7QJ98qLwwNAtY~O;UoU@Q9}HhL2{sINDee%`%^rWVTW3!xD|^p4CT{^E9NaCnX*) zydj_}yzP(bp_xz}j;&gsZfI2Zsy?beH>i>lRE0-Wg*SXu%f-=#WK^pVN3D+|G@83t zAI;kgnxw?@g-6d9-u6fHpj2oM$2P6c4>X#4R3FVR7&J)*9HHdX#5UbMS{JrVvASLq?gtrlr3vc_QxOWnY!!bo`8ofqw=k0)Ejj=^i zf}-$Q^YApMdR!A8Z|DNY9n|Aws5+b+>G>E&N&I6* zu~3*+=qO--h5&&mH!X#hgj{+fF+Ar)vz&e^QogSrr#-qf8mz;?yAJ*YX^8jr^6og{_UW6>CIKnWyg~os9H{Nb^XaM|u|0A0a&( z>5q|~hx8{%FF^WVNH0bDQ>2vtXGo!J?*~9lbWci3Ea9=l5}q}B{X~EJ7#LA7VEsN^ z<<6R=O~V?oJ|%V^3T%5yaTx}xr?vOAE$S%DDfDz>3B>~A>4GArv@5i6gE<1U?Xl~Z zFud!>05c&YXYLVy8V@74?HtuZCF>3lSY#z7JA4U`$O|tL8TE`ie|IH&LgQYz>sFvv z#dS5%-(>nik|+0{FJ zReO(EVh(rdBZ>_lw;4msJ~rGGZ6a8~%(OGxR_fVlEM;;y^}&3#Vvy0s41@a$vmP@d zMoQ|3!rO?ugjYFcy&=rj;fbd@rLv7#u^eU!EZHh8hiBuC+tnYc}#$o zAKSPjt^bix4j1Sn9-i6t#+df@jcJ5p_4F;0Mk02Am91WVes5dR>=v8H!-b8I$K7I; zv#NGqPQMo^^Sc$NZFv{@XnxH^CQ|ZonD92^4&e<4&7lyQ_u-kxB%pb02{dP_xRt}* z`uO+3>Dp!-nwR)!ZZc?+5;TQJG=&$Sxh6D0ATZ6*55<_~5qQAYLtx$oj=Re%%iFOy zSsol$0?g*N)UnPoPp)uH(YtYSr-tF1CG+Z<{ z$`5kZH7A>5HV|h3-uD4c#Ack_wlmA@9H`$bij#`Oe9zkjH+WoF!gF>*usWQnAi?L#BR17uJE#vv*yAL*1CaLa3>N~uu zAE#a4L2E#H>2avyN$Z4^d`u*~n{kKmO3B)eD<(L_ge~i#<8gr;rY@)hkp6*byj)gU zn;@54U>@Sj^$DXxQgupx;a!F|3a?VR_ywnbR3KL?AhO$%VzW8@M5M&fl?y^BeX->y z`XVKA36F9K&zDQ~?3KdjnUF&c;!<~QR5lT5q`TrgKWIMF4OJn6Ra=_dJ6ANfFDkYb zSF4gl@xhpOD%;blsmIB^cz`+LRr{-b!w|StO6Da8Yqbp)EpP=uC(0H9fRs%66W;B( zOL&NWT6h-1SB8XN49+Qz)gqXotspb?PYASo>Wmrlx>giAjkLWpI~Ai$w}B}`b9kmc z_>O)|aJPk;?+f)Sqg7IM9s{}XSZWH-vev==oAIC^=&-BB6=@Ifs84j9c*_Ejq`aEj z+t!O-5jf-2w#CZ}`F8ByA{NcQvjR-a4N$ z^Yzgrtsm@P9}<(OxaEi~=JGOP&PD&Gzo$8#WR7?nbp%Z#CIYAubEXlKl!z%jiYYu> zo3gIR;838@0bJ*9p9M%ZHezYj?k&Y{h}$mu-2q(3hP>QlB6$~_(V!9T6nZ*;U!CY4X$(N{@Ie= z<4Q{1Cd{GEF1|I1x&mE~JyM4*Qqs#NybX{{c*8^B?lcHkWBb_=uqHlNfWVOk0aAj1 z@Q8r$k`cHG*DZ8PUfTvX#3GPlu5xxLtjWp(MHX*3-ylIsdZ&fA5fTb-cu3rt35j7D zg*CxAI}%qKBuEJo!XpyGOGe^aT(=;>Chkf`B06y>yDeCgg0mwq-5@|pyh3>N3gHb8 zfzPHwU|2R_P2|mvz&Qp1Qi6c+h=A~t5x5E0CB=o?OCnHa?rnBkuqNMTN8+0X2~y$> z!lO3`Z+J+2IujDZvI%R#Y<47GFi4ORB!ovKgqMs&xw!C|WF)E)7pzIH*%A1JL4cHa zh4APV!W$j}ccemKSTv5Oxh9~*wk|ZCN%B=~Y6-d5=(K#uRTzHgRc*&BN^ZOm$ zHek6qo7u|%Ytm+RtN+$`-A*7SRxdnSz3`F|=*M;L&abpK-S+>KtNZ9Y%j_~+6D=uo zmfA4;%cMh&l(d3{$5ycLk|pL0A@|F8TjE+*=SE_7d(2ziU62m*sco4$JFJP3+4Vfd zNKHz5JA}vH4&nJy54?`H*0Y;(;LJlOT4J9b-^;Z*>~5clez(s*P1gV^ z=^7B;I@~2Z9|+wiyAyu+VW>~`LU7z~(?^4+JER9J9EVxp zG3cF!a}Hvye?)?w%}{9O^rc9{fsadV@k!SzG?x^*n>pRls_!7UnS<|l7bf~C0A-#4 zO#jJ9Uqs6M{oDt77t=FLN?b;GbQ$553x3+U-aw@% z{O_Syon03yccF>czQXsv@+Rgtm?yLskYk=uqw7Z?^4^S;`5TKAQOLXSP+h!`5?>J> zeMNZAfbRQMAkjdagUehX2plx1mR@^D0E9hUKYe9)*>N!y5?nvvv#D6SHiRIrg!!Zq zhE$!J9VonB+#x*65Bx0y>aqH8;0_(Q(p{exV5uUL)#AfT$m374#TDj=suk+8lE?@1gqlZ>sMd=nMTZQ{N*cLJN;V3(wMVe$)`yfsR+DS^1t z+KYw77?fYl^4ElRmRV_o`Q8mCLdBUfb=&GpMqvD5+*x46Bqeou;jQz3HMPEhmsR|3 z8vRiJa+oRBdqbs&jiOYQqQ_Y&YRZoq3%AP;Zl-H_Y8mF!6dvcaNIe)*MnDwa}eh^gfRb* zPi9bBVB&sT2meweTT_-or_n`?7Uz`M^~Fn zlZD325ssx+Wie+ik;xe>0PV5?RbM3`i*J#TluRoWUO%27Jln4ZhM;g=$i;aTOi>BN zcGOFOS52MSxwxZmNui~!*y9)bJ}0eLy>xI$k8T0T_qI9_Q+w5X)AvbAyh?cMaF_7H z)6>PZ@`JU?F<3v!OZ_PS(MQJOp8Vhb;d=p=>6dG`oNN&;NP9}^^`b-?iZJwB5) z@ET0^^}PRuDr(e1ztT=w0NypqoZbUU+=}xY!|jxhgX4bZhnrN3mYQk~Ow6vKsw&t_ zET6h+_R^FtW%bRl3YN#iI}z1pk3g#hpSsDHe?P=`miY3I50+L8HP(bvAIuxuF!Y;a zeCBm9OIi-}q811SnEgHY`$I@6!wpCoCp}XJl9H8Xh4-NUtBI2UxjxkTj|)2VDea1l z0Zw++thvy+%wMybuh2)>0~vJqET=XT2|B!nW@Eb~N)093$Bj#plAt5JHMm1~mP-!y z-wqH1r@R;!xeHQ4IZ8Y_NqZokJMXT!ec5uEd^feLuT!E8?l4EZtQAa3JJJ{RbfZR6 z(r^*pjd-c>EK&2LHiI3A8rL0Zi@MKIChJqXR(7KZFPZ`!)41E0q9-e51C7g9hkC2r z_`~;3bb{0*kT}T()`$2h2pZpziWQ% zzCHk5rW_?D^J9f~JERhxt!MPr1X?~1m$+Z|(U5g--8q3l8|`M8_Zw+?yPD;@+?My} ze8C!wW=ToYLwJ|rF5%gN*|YKoaD_Xcl5s)Dq5CG_4>=OYlT-lDAn<+%g-zL&mw8|d z$&O7^eR=<3>ZhcnOGS9M;4a~nlD8XIxZnQU@}4|igvI}+e3CicRBK89WJng(b7OXH||ot9!^|O8;zWh8!2A+ zUU$AT>{Aslrx}waCBaB|>u{IwLSn89iTOTUL zM1}g4s56YHq@;Ke9*Y;@S)!s3y(tJgte&{L8QNjzSAl!RRBq4g>Xc7uvXrDw6*u}3 z2H)>ia&;kdmZ{{Dk`}V?ZpK~0^M$nwnGyYVgX}6?I@g)q!1@?t(yln)q$_GDi%@vB zps6ec>ymt-ZAqah^Y_(o(A18$&Sh*iX9!KFA?KF&Mgd@6>YZ9X>X}O0b+9~X%8-&m zMtCe_gck$A|AqQRF2|+r^bQ+<9n%9aT-hV09w)pW1mp%CB0MRXb4YxyAK)1<_wg-4$jo-ZgrHjG)3 zhNW`Q1<{SW+)@b^g(pkH`91Nn_x7$fcAdke`iKTG-tW{gTzUCEw}x?_FYkdyUQ$vj z36G_c@I>B?I|spwQn}r=l!$)JEY1(4LYE3+GWK=oUCmhwWy)E8QC^P+><=By98%T3{EJ;O2d0_$Y-C+ zhNPzkEOP#U1%5ev2wgIMC)F8{x<;)CF~8ro0oK(fIZ^LS@OHzQ$9YiT^mE!emvqTU zFiF*JPT6X13xXS)iF|M$PHeQpo@3op7$}B;2nX5}0MTn8jbncx$b7K_hmSbGN0hC5 z2>_&|W+pt=%!C&c{nm%t7;%BXj($Bs?xYT5hWO^7ASf;=EScTc+ufJ88e^@}B3)Om z3Gv;nHDMDv$rtz%V?v}P5D1TfKzJhX(L;e_5OKGY#CD9c0trp1n^RcYfi7X}fHJdl zY1dh0JVbW^I6jZ}Qo7*r%(J+?tzPy#(n*g)T8s2}U)pg-T2fML6W%&pCA=W)Z3T~A zk-3GB?F{NoN7~kY4LI&sFMEN<>=5u-N8>*^2tlduT~xCp4w{_48!1JQL(lCY01x4V z*ibPaDe9OXcL$gi*lmb?KedoXES;3hW)t29|5wxZgA%}jV&6?xWS$8Zd=8m14X$nV1|JSmyZgWhZ`2+tYiKT z0!f>Jgp|zk7T!ikD7=!8T#yNpkEevB3TE>2N+9{D56OiF5>f(*@Cb?UN2$E}Y zgZpKSZ6zy9LE=n+9)@+?=z2^V>-dZBFvl7wNQuJ;j}9Zel2DwU35tu$Kv4y&7+Hx` zH279=hJk{VKp{LrA-s}MJRE|eTW zZ8WaK#V%2fbfBo7F+AVs7(=RBXq{h@{1>EK9w zcMJsc4^*Z556HP_o|H5^g~x`c@B*YLo4W>!zUW4&?6!$L)o};2EAS zO9ieR4?+%W4%*y8Z!cC=lNl5_eF4&1L)*4^s*{D=sxYU{ z6`1dKKcLVPe?DTYn3Pzt@My)tvoso-K>?v7EBaCalyf(nb?CM5kK~nLGR&DQw+am3 z?|%FEUSGK*OrRnqLnFfD(1`F#$r?8g+?MsZW3;HA$VwB=EyjY*y}3i>6R-?YmS6iN zNzMhvh)Ib~g-4wVuaul|vtVsGzvIfu;SGCrpUkpOb|&UUM|+ZF{hpDPl;~7=)T!`F z$=Zu6PICHtZCQWh%IePp$}D1;)oqgGdlAiC2}qvDATkf4)xy?75Q@c8s zwlB*pX_+rnljNLaIgs)FPhZe$Pi2X6*?9-#Z0w^$&ZvQtB9b+CBx)N3?#C z!%x<<$N1q2zSEsu0#&G5ZnEPbN}h0c2neW*{+Rv>kkig70z%DB+GuumemWCM8p#gm)S45}qyCz|h%&V5@M6J1yE4?A&<4 zdaLn~RT?25^o2aw2uVtYUxc>~5(v*0a#TOqftV}t*KtmPvj=#rFj3sDKuo)O&jk@v zYS5|MoKCu(47C|i%I=J1m~m=||y z6x!Fyl+Q3Qkdm@Tcr1H_XX2DKw_x-`L-P3rD%Tn2evi)&42_jhx_aFB5xld;YXJ3G zH7JWey?Ycz$Yj>{Nua{BzT+BKXW=U1-GXyffHzhSJ*%zQ&M$i<^z|0edQ@oVaHT$? z!HSOekgN-Xy zk5OXsFMs-P=8u;Ch2q2NFm!#2@1N)hjb^kmrVT(%DsZHv&=VdDJ>kU&{X3=c8eH-! z&I@t6lL~F2w;Y8F)mnCZ?fbhMoh*Bu)RqdKCRGMD7N($GK=k~7A*(a{ez?GtG9JY14ASA7AD#6L_$Eyb*>l5@AmKUlMA;bXglx;_-0I>B*2a)x85Y7HWTcXkPB z&CEGmV~%*sjm)BPCFc8Fi3fa%b4FrPQmqjlt2M$~)7((IfHfLaWGrDxhJ{@Ljw|e& z!Q-qmc2&Qta$&h`3*A(zFaDwruAqFED<~AF!tee@P*S2=;Ze21Yg(}JO@X1l7kszF znYB?w0gr{x9TV`M98)^l+pzzZzs3Qrb#9p&nzW80ag@(zh2JhfP-{~X{5O0}N=s6* ze}wROhwvalT_C}`!FTJ_wgjIEB$zt4g6zBa^V#d1AgO2H{@Uc8lzd??JcfATA@{ok zxo-vE?Gv|iKQ;*ckx!S>svu4R_Y);ufd8+tAX0+A@QA@Rj_d5i+FM#hho$cJ8YUMumllHV`g!jZr;(o%eE$M6{2`TxaSa|(-lJFpj z^?_G8JzN?{g7pWgqlY}gwrxd-;L3{bGr1=vdJrDy6AVC}WZX?E);I2Ri zbFi4&1qCdyQGpP#Qk?Br5}G+mikFQZNXgbE!n+u`6&|FpKA0%y#dil%u!WLYfIs5B zIh^_+Lac1rI?!JHJCl1-;s?T`9|#Y*-zm^Rx!T<4g4|m}J(xZ*%rd~8B5|aT|DTKm zq{NDZM=KJZ)zrw3dLHaR9A4xWZFYC*rvsIzuZJ?QqnOXysL4S@IpF% z80I<31HzO|_6H+p+b4BI5hj7}W>KQ*&jx

jauyeh}O^ATJ~WPJV-tZVq%Yp;Fq z0xThNCG0KL5+dK^-kF$PWWNymTd}{3FEB2yixk$A4JpgmlCD4I}U^#NmEyfSZj{qrBz>y zG_X8wS;;P&Zg9H~GL}bHBUrLBiu&qn99TKSTHIGGVm&dlWH1+3X~M66M_k2cV#UQ+ zpH3Eult*+HE;QHffnVU7{d zNu*aJeH7_6NS{D@9n#+-y&mb`k$wp2j(G14NNbVai1aN;KaBKXq&Fcw8tF%nQhzri zMJ)DqfY!xjN!598$8W-;1qg4l;WZiF|94|XjVieDStu05jXCc~+<4Hq@pXxA{BCte z*p1IoJnhCgD9l}m+IHhFxo*t*fjuiLy#pUHcK8nb#|#cE*E$X?OxuBl6X(FM`VPzz z?myLi-vw=p`;w~j#zEi0qvnNojNwf*y#E~cT`^Rz%5AK&>6XH?q!g}){_IkC9t+^+ zhO-s#=mwN=A=M=RZ*o1n zU(Bj^Z$pz>b?{HO>$>;YkgKb|nznsD$OmGJok^2FT?zMdrR(A5ProX@eBrr|(U`t9 zo^FmBteSW!4nL%d?l$nguBfG;q%PhAO6uZ+pq^Iuz646@;#B4{il^QDC~#VwSyAoa z?TmXCA^nQ$=2qWD{4`bV{LAqKeZXn-;aAK1@%Ld>sFwdFgY(O^j`IuCc7Ea5&i_l5 z`BC1Rkzs3GZKIY6DKU$ymlH?U%h5Zj>gC9;s+Z3|%6j?xk+NRSMl9>)wSvb?}>7;b~X-MhzbCJ^RsY~tlPr(Aj?Mc;n zN8mT%(Hex8H@xP6x0d6y@cLF&YT7PuH-5K|zuRwsvnMyN+lLy_J6x)4f2X)PFEKK} z!^@QIqtqjHD;^8->c5+|eR|mCJ9op8nmYSjqHINcgnj`5bV};OzrLxlr-0{;Ebzo;yH^r*qD|c=`h4>1QW;`ZRUNvyenQ{a2ul zQoL6|O;8kT^3xTiJ)Jc{ceOyfCjZ5fo^F4sklxpyKp)g_ef_r?d|j?}d|jBfuM5Za z^(_F6y=|M3VXIjGPp{Qah4#hUN!59@2;tEZghwA29=qLy*Noyl*Ng%VRoQu%ils~Y z+|fe=e;?6{KJfAUth?H`D0V%H; zLPx3Sj#QoZl;P1>g(r%!rvY{1H+Nbb%kdp?5`niIjsN7ZQ0tjO|1L9)OwW3JUjSJk zZB9QGDc|R=76ol8sWus@NXZ;P;jO`4!V{?`3Rht(*VnMo}7 zBXsI8=RcGQ-{%gnp6E;Uw6a;AL`u+oBP4YTM%F1+$S@y}k!2 zEkh3N8#(-Wh#$wr- z7#)k{#lcu?FK&0|Hyi*S!C=ekws2tVLA5rNUq4o_+@0tVO=?ah849duFGQyMc zKd?}|t8tg`jPQXu?2yhJuEaCkcDij2pNK~wVh+==p-QoRk+rwn_AvfLuC)g&jJ|2*nJy8U5c$7Y>W)^7JD*!%X03apv6NI-8 z?-ZT^K+IxbRjt>TB1Ejl1KcJY`xodHEO@_n4E~cN_OtkzJ;7F2<*lInXYqVDn?`nM zz}zYY!`T=@kkCL%hVO-UBkmGjg)qbg2OEaxT^NF%>TC#n93UmkQb`j3=Z9+XNl9=J z-qjFacomAj5)W|aVB6v!G9DMmVYbr9CbzqhMs(r^-?tT~t$Fm7Da~`YA<;Z3DJF%t z26qV0h(6vLQNibYbx)$uz~75loneIDywn80wu6W-%KWv$bT;HyyVACHD3d%u*O0ai>qYaU$NNUqSAHpZBShK7bCX+lg)5;cUzM5Li1 zgeWH0af!mwO%x@m-m14XFcnA*3#;yGO2zcnKdF3zC@EgxqJOq3S?3?`QJF4aVgsw zrQ;@#mnt-IlKdItg?E0MRfWtI_L8h^P@>$+G|9(Ej6B$54JtUxJ2%ISHA31aSJ^UX zsZ3){xUA#wi~C|+&IRfL|2HbXtfeCoD{8rAlH<9{1W#d^=2&GC(q4wj=0Hnj<&15a zRUQ=^cZa3QOC;&d8$_(@zZRLj4Rbj4Z%EJm91uitUJOeZxiV~t?!C~c@}=c{@MaF4 z@bHh3Iu|?BtrKp>123%HD`^S98@=UpTo4hayK zdD8_lM3*@V+fTwaDk>d%0ovg9?RC|{=|e}49yauXi)|0M&xV5ek@}$@i4Ec{l2Qo< z2zn)55K^_p)a=mCvXSG@K}@x~3t|DQ*4vhmg!CZ-Dq9RKl|`yr1)(rnBz4LeE9DC9 zNOYXThIti|A+CGz5*{<~A0tejv%|4crqV@vDE#@(=P|Z#kNNq${*y-y>67!r7LHU4 z_aJ+rYaMx+U;z%VwQy)@>CkCxym~_xs);i6S{J)zLzE5w>y?CRj~9f>uB?=0;;|IP z9goe2%h!fgv{niQJzrp7m3pZ!Ou6AxgNW&yLIRcgAS=Dy{M!m>@C3pGi$ujETkY=HvKhS%w!< z!>ddTud)Pdd>4n|Uqf<3qty5bkqqy8#&VU^^oJ6TF~5`-V`4G5`OyDUErSbbh*6n@ z7?mYx@Bwidd{zYpuhJUcIN{JTCehGhzFMHjGPsZ$TxDW#l_hBK?GA%i6w_ZQ$KcUk zJWwSyy(juhiG`NM)J!0>m@hEcVi{gYLyO8Jw5TjW!@nDs;ophT@KsvlA5Ylu)zIHg zs4+EPlkmD_cp){s%Ea(0OVIF39fq$crvEdN;SWVjmlyXYR?{C%v=KF5#&C;ea3Kvb zDw7bSvIGtOU|a@&vI2uwX$^ljVS^u+XlOBC`Eb8wa3M9g%EaI*OVHq-!~4QlqAw!3 z;wZhA6I}*B6czi2W{Fnp&b)SYh47fJK+P9Rd<1Lg6{t|q^C9e^vQI*kaKAjsQL`&D zbm#I7U8NQJ!h|pN*CuRe^Ys<~wG1t!hE|yvT4m1JXdFiwBENcMz4XXSl0SW47-Rof zjMAe?=qGdZdPrzJ2q&cvkwLyuo91wTth*@iZZEiw3$Qs3$T^;zTi!~NZZ>f!?%4P)Xe!+0V7lmSFt`=zVGnqlCuK*rOX6(sX`P{nSt5ZwB^-RC!n1X)b8 zgw#?=Rb{ScE%g(gEf8sp2~X+9sFJntS(ahi=SD)L<3|?^ z9glIoNz**CR=OF)2a$7KY%NQwvn@$zlBX=7mg4Jf=9ErRZf8kl>ZdHJz8VslUrvJ6aRwL(kkP)p@F| zNKUtEOqsY08jD+*@jM~7>cwkwjvQ9tR=*(UPPbBt%dJ%Cb}N;--KsGT zZg;CZ{QZY>s}`_}x|NW+mCD4eR3>hvGCAWa6Sq>CxRuHdyOqnWCeU3ZT$hrZZZ(`T zajS_W)2(DZoBLI$_SWjza^0$;@#2L1YiW28rvm>*i5eN{E6US=OX7IrF-otX5T-tZG!*VfS*m z*EMt-aj!Wfr+ZyWnYhJ%QlZ(h=beR zY6brOqqSP6!z}7nLP5?AQJD{CS7m7-s^O6{t}^Kq>VDEGRM}y-a=F!cbQf`}fh4C} zokiJa_;nB|R=0X4Cbz0oYt;e`L~L#)b2goBr4pB0snG3KDs{V+$JJV8`~h35{y3xR zRzm7lDigO-nYfk8#H~~&ZlyADE0rB~E0T$}%t)3&9ZdK8>)^icv zD%ySAgjy|mGUOUsF85mf2XwD@aAwuLgw(xMChny&aW9pLd#Oy^OJ(9-Dm(06 zF88{fZX@n>2g&JP3n&x!x|3wOmvj-LU2E-#?p{@Kt#xV4Lm1vNZlw~JTdC0PRw{M7 zRZkq;-f9iS-+#1L>u)%t>Q+M9DO8zs3RNbZLY2uGSDADQRVJN6l^u2~ms>qVcM-Q* zM{>H=J(P)CttZ9mRy$*It4g(2pT_J~zA|p55|>-4(Ct<#b-UH$F1OnC2Xw2oIHT%T zLh4p36Sq>CxRuJptyCs%r803Vl^u2~ms@>-2`hTLcpu5>R{y3<-0Dk`=~gvh(6d}K ztL!6#o6B{ps%WH)m}g)77nQ|1a-kBJH>uF=O)7PJ(;m#~us2el;%{lBI4f%ztYm(E z&-bxD!(!50A=Zu7=8FwuZ{&Osg#0k~Ohphuz8LP6k^aX{Ed*r#pRTxs#8- z#_CR6qPbJ08m9>{yVHR(?xYfzJE_p^PAYY~Qy14{rs6;1WF`;nqV6Q5?xZqtCzXji zsZ88SW#UdM6L(Tsh3@n+_KataARq3EnRgY=5#+;U+ijHnKfy7$Zr(jdFf=7lf7dyI z=EGynIf3Ti?et-?anA|V2gr*1=;L6W?xT-`y$8SQ<6vdZne%b5-$T;f)h-=`tQP)D zocm83eCbDL2bO&tZ2S`fmtl@wY@>ZLx7pbwsl??&Ds=mhO5HwWJ>j)X8zkjElrsQV zi5)SsPi6wj48Wb($_&8gv6UHs?_wLk_H%4y2B6sAoB?^|7nVdnD$xwvK zq;sb-r-Nj3VyjF#wLdDOK2oTGTMFTM;PWAIKPsaMWKt^{;(s@(aQbmRPsd#svd!JF z4OnbG!a-6w^9V;>kcphG$)W5olzp;;k=m?iUsv1z8RJm#2nY5H^D>%svB83mI8}760&PTX+)t*%l6yWC;z(@oZjCSf*dJ5~ zyL20R@rZM#O`J4hOt+zP2cViYL?uRm_LnaDo_nC8*f-!mjX zU-_K|IyPjWuTUDT7((h9DihC8S(iC@B(5z;T=_*S#EAlZk|&;uyBT}N^iSv!36GxOz84>GKN_x@a)}-SCBs>}aWdUC(ouhDWdP5|K z>ym}#U{762fzKOrVQTf?~;!ECP=2?ZZjDk_v%t1PG zPRyr^v(HBT@LCe{H!dUzEgxwqDPd)Mds;k6;X^*3uRNVCtM5sAEtZz|fJPnVd- z6mxK;%hV~g#3iRPCn}|aV@ZpdjdtZ`JYNoxFGXuN5gEnPvYcyM3MB0UD{aS8m96&d zp=J&_|7o$bu0@gFwX7<@gUw+;U5nBZRGl(2dde$k`nZBcKgeo9yEnEI%Dq@)ceB@p zIs`KDe0>Gw(9wE(SsrP^7&-)Ln4f?N9*^xMfVR~q(xy(uPTHpOC4jP|BpV`qso%KK z!Ug{eO8S|B&B==9Iu+elyQO3J?38BM z!Ir3j)>4K?ZcsfgPLahc4Pg~&U1af!c7OJZVrVbfi2bCmDlP5SkQ}|fSz^=L*k)mi zfjzzt5N~3fg?nxuygGoZRvKX23xDG%vo6LK=Wf;|*fzrUCTtsH`*&>N6tg6%;Bsr$ zT5Rzc@mUXJdpx$!V%q}SSFy$0LuN_-@X8^+uHZy$Wl*s#w(=UHJZw8-dlI%?ux*E} zoOissfSxQ*!_ifOF_3jwoPTEE9JW*o|_yqJWBGivqf@ zI||qmbw>eNOJk1$vexF$9|hzQO&SG+GzzFpqJYXI3aCt?fXXBas7#`O$|MS?Orn6w z4o3mL?C=ou1t!qXB?`z|4R;h6MwuA}&cOLNFG^ovYPnG$w)H)6gag~!p18xoD=3gF z9H=O^aG?9T!+|YPcQ}xxKlX4SOM@5={>*)X@i4MR1EHWthK5xpYhP7{env=!o<&G@ zx+RmEruz-FWJ4_3;b?HEqQN!vcZmkFw8R|^rcxGbG?*4wG)TB}U|Wh5cQn|80=c4r zieiffy01GL*b;R|16iqLj|Q@m=}#XGs%kZ0pkEjs~xz zK(1(@qS&H=?(2>QwnW|0K$fl9qk$}I`_o4Qc_yt!10lUKP?=mAs7$U5R3=vjDw8V% zl}SxgnOqsDOs))6b~qXws%W5#>LeP-vM6^n*hN{a(O^bg(IDZzfo&OG+|l3-6v!0~ zR1{k@(0$#}z?P^x8ptX;do++$cz^n6@Ggw3(LhL}fyyKrs7#`P$|M@7Orn9xBpRqp zqJhdJ8mR1WG&oezK-U0DH251mANQnf^9in}%!~#VjT2lNJsMPYoWQo~FYYMtDhlF? z0xF6v3h2J>C}2y}9R*}jpgju6qQO6X6j%x)YZMUDD+HCv6@tp-3PELZg`hIILQt91 zFqO#_g39CyL1l-dz@dr)57Eyh3alkLqrh^?VvPcq#S{e+?hDu!DaIWIUPD1#Q9wnp zMFHK{9R+NOx}$)sVYEj9S>yPpj{*_sa@VyZ6fq(m%iVC&9|<{X**fDiiNlnRvg-#QRky-mfz8ewB&$t4zFK zWuf<=V(xHSt%qmHoc-_{bFaoJbF0RLR~Nd zi!#QK8#{4Cw^5@<^qr_P-JxoI!ZY2?1u|*)ZB||BRz`apvgD(k>vgA?49{8E2XXUA zZ6>5|2UgiKXsOJ4|0|a}lW|F4mXS--;A% ztFUFgi!S9tQ`B$h=*c6(*NEBp&HPAyenj`%?l5?V2^~gOh-ys`(q(ljTL~?dMS4*c zCMJ8#1ydEjje9B5YwNPqsgNSaQa6`vP~+JG zNln^et3~pv<_;1!y7y||p4+h(j?2zLRT?$|L8O9ps?zTxMT@#&uHMrEHODIgB-W zl8xh=pJ-(~+d9WWL60;bDq9eu$Ys0?dvu9X#@+5Ry3c2vXR`yMpS{s^*X8oci9+#Q z?zYM)6!b{xRJJHYk;_?%Jx+;I&bQs=40j3QsjPF5e|zzG9h{A0o1gN^XV%r%t>7=D zT>_QK8C99p)@d)P;L3KMNg2*gk-Vnt^Xdi9Vk1YDAG3AYZ$tc<0@H=t!ts!t>@rJ7 z$b$HP68?8?g)jX$AI1psVb?y;BI9ou6oGX`CFj$gLQETrdHWpa*HR;J9T*6uQo$KLLcWk!%HEaWC zI}p2#T&+NCHtH6#kukrI354fHrdFH`;84xr-gDedt(fB&*F$P;2N}};YRE_g3 z4JFYySVkk$%o4UkEDa^m@SaBhv(?){HjSZ{hLUJhR}CY{qn;sGy5Y|-OG8OCYLz*L zy(V5@X()+Cov0eaEe$2nI4Y{f2unjrG#Z69`e;2tU)TCvXlW>k#<68IAh13cSsF^B z(Ly!q8J>E%by(lAll2*CX()-tiBUC1SsF^Bak6S;8lKGD3?4%+q0CQ-rJ*Dmr$s%+ z#g>MWXmnMLBMi?GeQNO-e>D$)?PyCwNi@z>jR;;l#?nv{jk8t57O(6PbgZSJBpT{Baj<+yx>U zq%y%w3fodEH6=-H9QPa|dr4fM%0DHpOS=(DUwW$MXQGvclB97Nm6B|B8f@;4?IcS> zNi;534cob}$HU2%hLUJpQ$|C2zvgF(rJ*Dm^HsxEjxpx$*j{33D2c{RWi(_A-~3Fq zG?YZ+Hr0@lhkX1>b9Zc~SsF^Bac5X#p+*IH8i)D0)Y4EAjTL1yUb1OiW@#vi#(k<` zvo3CPS)b{ahLUKkEu&#SA2TcsCDC|9HN?8*v`;fF4JFZdBCPSC)?2wdXny``X()-t z)2d;!u3YJwpIMfMl4$G*YlM19t<2rAz1-4J5{;KsBSLF)g{7e+8hcd3W?hVWSRV}2 zg!-ULqVbk$*m`3Bvi8TgyQQHd8t+k~cDmt7-*5IY+&;dK`$$5WGPn`?Ty3SMB&q#} zdp0w75vzYiQ8MToQbwx$+A^Lm1PMa#$ZoUn%9%oIc0o}-q;%-jHg>4&6ZpUD2WliXF1(ka!~W5MG!3|(K=9}mKImE zltjzOmqJE6mYPPilte2~p_ZCaw3I|EwOlQXCs`jgxo9bgRz`k#Ikao(=@Bg@(W+me zmYzz{QWCA~3bpjKinsf=(HI+5W1gjb5&bfA7Wcu_COKoeK?4`cJ(ohnOd0~xEmk066`YfwHA0<>nUI>Fn=B0_(O8~ul{{1q`=#t6OG8OCRz=mg z+0sxFjWuO7&VtnZ++t}ciN?eE?p)b5ZnZR&MB{PQu+`fpkeZ*xmWGmOY{};_Y~Iw! z+#TE7EDa^m*sdCKj8I2xvrS`(rJ*DmFRF%IY%NqTf7Pb(H%miFG+t4SBk(Zjq}m>| zRYOE!RFZt_XQ`#7BwBCgyXzL(R_k-SrJ*Dmd-E*|=W5r=exY)QrJ*DmAFGDVUO`CB z&z+Wrl4yL9@3ydAW0|F)BpUn6Xw-(({M=<}D2c|8`R-%bJNb888cL$^i)!Fm@X6!x zCq75&hTng;G?YZc*U5bhd#;vS8cL!OjH@7dg}5YHuz!SQ<*AaSt`3k2yWa zedNIQI=9hEO-WL_pL-6GU6b&CJ^z$Wo-Id1<=4>XQ7a84Nn-<*?DrDv-RH+F4JFZd zvW$kkae3U*P!f%;s$uI$_6y}mHcwa@N}{n-HKZJ&F56W$jVCP)CDGUw)(CY&+9icm zp0YHQMB`P}u=#l1u!3ikrJ*DmZoxIu(vym; zdv%iL66C>^$2^gvN3qn%@{ILOaiwh=Y|`{vI?^jCH;+pVB;zP+?Ht3Cu`n&vRkh{8 zDp!f|oy$J4=uC6z)a&f^9GR4o6h0E>*Wd(LWe$eV%U~&ZOX~}v;YFjVKR?hsIK~KM z`57eb=O`Ag*A1h6N%fQTevzb&OSG7ef144?Kz^dfPm+tY(mLb0?GxI{-x;ufpS1Dj z-HqDWsJErzyC{}@tn)G4=!3jF${h09dL5-+)wSxuGVG^yqm?Ct$a87Abz`ehgx?(L z9*4VM3u}!pFOAX&X$)|sq36Ag$9~?I*mcz(&AOgsAQ$a@bDa6G)phz1S~AmKW_zoC z`ia9&b1@8T+0W+p?4=L(9b>?;9Obngr@ZvqM@z5BH}|(rFKT&d2{H412UQOz&EQ%LHqmPD zFvsuiztM-_@21?`n(`R7?2Eh{t}&L~oUw0$*J$FYY51CmeIwe&UQf-um|HV1r4h9PA%q%u@S zUY?&;&`w-lEt;RQzuC&sj+NhDHwYMg8#vEPs2|{{&t{#7b!)35OsBNsx)k+zW_q@5 z0yeAawsn3Lx7AV8i)x=fF8ku|s)%{z*|>uCBZmIL97nsCi;0YupQogX=hBnQ`@(Ru zG=r=CIf0yDPRixgf?3j)W}{FU|Fi3AnbRD3EDhN57@=+6>7U^ZHO)?cvh@cmf1I(t z|IKj*!^m@!4}E!xP@W0@BfBMXoL3z6U?HlfRaf;|lT8il>TIj()i2SEp0B#$uP%L06fTbX-PmEj zV0GJZY}e|v-%Pvx78reeQ~d*RylF<@ncyfGFG_o7w^1ZkzrM%OCN{<`8mrt^-xZgS zpBj!bs_sgMZf?1{bt35Qbm$h^?d`s%+~n)zjZ#+4%lD4-?zUf3mUI3jXuj;f%ZSIH ztE8^6AM&814qRK)X7i|ZV74*ZSL8q0k6aH76a^<#Lmjxqab6;w>qv67UFGsq$IMU9 z-#$OnVV^3^5BuSrt!rEHNCTH(mXD+&0NUHxh&OyIikp63^2yhayk($*&yoqtqTh-0|pJu}kDJu1NjT%9dTv#}w;)OsR_V z!LqM(#H051cw~#6tRtxIvyH2K$M{h(lKfH~Vv0W+XFhY-xGU5)y{B!5``+>SZ;pRV>r00%O6;~M_c&kplB)aX z|1XZS-kZpAqS!6cxY%stWCKyQV<0f!92m35o>*AxK;p;#+Ni{_c`s+B<6cfhzPQAg z<=f{o_WF1Dj3qJpBIA>N+<^Vx_KyDV&a(FsSVb|$zub7(H^yf)jOh3EnBHfO^rHCs za$~FSqiAC{RoCyT!uuF(TbnuB*7k95%) z5;1N2G4(jk>$uL5^F7=6-nZFTVVfRDdaHedD!wj^BfYB5-^adDRb1ZhqLtUYXImw8 zQe%muPvN+x)^W)^sVjr!Yqor?makXk3*(ZDjHRAaeJed({P-Wh{~-RStcBy*vL8T=O6l57lt}lj1%ChkcqtJAP`%?y_woiF6 zZA$v8w5*gY9dnJzXCjs*!kDOifAzO0<28}{MZJBq{j&oz%%~p!IjE94#Wr(|<6643 zj_rKd#=67jW3_dTP^T6e!+mwcr&{?31d>o3PhPMjCCMACY#evknfZLi(atn(Xv=5a zR4Y&6+QRGR@Sh#Sx83%I5x*}qvM>lb%^T=su#J@h^fgkud!-&3d1(XE>-f?D86|b= z_%gDLpAUJxvX?y~+wYH-b8jVqwtj82SQdABMZVjk-Fq@)LhWNPWAnlA6n85YSYw?- zjNLe2b~{u_-Df;?URT6&4+Zxe%u6&!p6vOva{_kTQ>Eu&tS^T24%t6Xo8f=v`d*21aQpwTeh&Awt+<~Z$93qW3esc$V4F69t(1#U)Hpvv#(+hai_ zPM{t6-_HBwIauU42Mg`zzT8Nok3BN#(Zc7?Pw9|)yDufn zze=x!BiW}ia<%P#N9@1CK2D?chu2oFS`)q+2^R(R=S%d6Cm6o5C`j|*p5#DYN>*xZ zU&- z72hX`BfTQuT~!)O$5wHAgMDqIjTOgzev7L3d{l6L%`uwt@)>o0<5^Wa4?d>!es+xi z?6luga^Djz^1V|d%Gh32!~)ZH?rl}1&GjJ&fx;^uR2vf%lBy*+xp86d7uYcI-%biX@M@ z+*qvC!G}ymDR=FrhNtHKa%`ygAK9L9TqhNCnT?uW7!qF{{5_Fsr2N_CYS)GiyMBiJ zSa!9p;U70f;aaM!Zj|}Iw(BkCk*5O_J!Ttqebv1fZNJdJ2NUjVV93dyZY*3C^;63} z+R(3U0o%Wl5_IEwh?~bCut`yA0FBUH2+; zF8-{#RpE~uPb>}1F^{xe=pV0s8+7<>=YXv~mpeCOk#CiM4KF}qIS1t(qp-qbEXOgv zi);R)S!O+6ila=9HK*m4NshNUfqBhN+r?I{_u?wopR+vs9qr_`_P$%W=Vz5~jek&F zW6jR`;5<(i*Kw&2m8;_wA7kwQ;23e1gX0#dj`nSsy?v{nDlJU71)J^s^B4KEXZVKF z!7x4$ZrjkHEi;<>9`&zd5IGg20weq>kFxWam%7T+B`quc(X=d2myE2sk7i(-nN|PM z%&NSH7E9cUcOSi0@#qOxPk@!K-)GSa{6JN$c# z-M`&+)aBp2$1O{My6P!xg>^DJQ&t#NechMnIbZGnU(~JPFhC-8EB^Dl+J9UEYr5hZ zPwQ0ic-pCsw)*|Zb1xSmsy`6zKDVCNIPy2xw>Db-5{hT$T!By=GviuSq?K5`m9={> zBygVVI?l`8XnL*dP;0DVfiVMZu)Ph4`y3SczB|P(H&*9m)FjE{XW$Z z{~pA{h%CEhy81Wa<~at+l^ZrizCNMVt~sd-({j>tGIHwXWX`B2SJ)i9ux4b1CNtXe z_j};D>qm)iRK;^N;*EVJFk{aIuE*k-Up&+5yVjL~v^iLtHAb4Xe5_sTSyk&7uXXrn zXAJ0AbP35_3 z>GT>d+W$1B~*AaJt9y3GixByV3!5 z>v=PG)dkeA=grzxAJDL#H+xq@K;wGeCc7E~n%46++tn11Q_tIcR}P?MJ#VXBEdjap zylr;n0`ls4+wIB&w6Evwu&X`b)Oz0hU8gP#0;3!ud?$?a@tly5v?G>mhHzgPZ)cf#ccUlf zmxOjW7{&*1Wg3zW;c0{>_MzOD{k?+sKAWm*zr9UsFI5jr~+qB~J z+%#-6itFZPV4GQ7KQ|NGtm1~bS=eS5H_pw*wn=f*+$PvIE6&MnhHdlWmbuNbZB?9` z+X~w@#d*1Hux(e|KDQmV9g0uQ?SO55ap&B8Y`YY9%k6?~L2+Sj0k%Dg&&utAZSUef zxxKM1D(;tCgl%#0fZSqi2Nn;`9f<9a;$gW%upM4}VeW8jM;4dlj>L9!@z~ta*p4qQ z%^i>Jq~a;Lldzpyd};1fY^N8`%$<(ytl}$jXJLC)@$B5Iu$@zUZSEXw=M`U{I}h6h z#S3#6V7sXJmfS_yE-qe@yBOQ0#dqW`#dcZo-MP!KU0!@o?(&s*^waNsyf2ckAC^at zg@^5P*gp0;plxGOioI=IGR)RCE;-ybnr+SRwrw0!PSaNyYWfO8O zuQ1f~6^5F=!cfy!7;5?oLrq^{sOc*VHGPGlrmry6^c9AhzQRz`R~Tyg3PVj_VW{aV z3^jd)p{B1e)btgGn!dtN(^nX3`U*o$Uty@}D-1P#g`uXeFx2!FhMK;@P}5f!YWfO8 zOEay%8SdQ(P1+Fm&T)XJ> zwVdD^7)->4H$L&6H$8qs$MHY2A3xF<;%p-i3kPxnFI&S!@gJM_3R4_o<$dh;3f=1f zT=xpYm$0O!cngBj9FN#5@;AV^t|t$ptbMVnH1!1|t#|qpew~jNx{(;Kohq}x+IW@2 z#+@sC#?TyJSD!J!zk<`=VzR9sC*EfnmEd_hMxh^Hv0e{pf9g@kdD{|U9W&1i2g>e| z>d8NOkM~30<%YkzjKAjx?lO$t!2_6|fwjR2)Zs|yM(y#W!|tEi?QZj}epc?7Ph7Om zHs*SL#~DVRe?y>kHC!&{2M-!}Y;pWPX3wA6Zh*sfJ2A*$jXApOJTvARY5d;GES_lG z-!Q(H#V@`6HzxbW26B8C;<+9v(}TIbhQU{S`Kh<~x~1LhO<#+ZRB#G=UJf;@{aZQg zUyAV^tE}$104|3{J=tYU^ZEw*n+N(D!IQlyOF2HhCp7D#s`9V-obJfyoA!A%wtC{q zBI?N^tY`0y(U`2@w7-N0&nuI+MEv++W2CR67sEp9P2}D{Qs{lgyvLr4xkHi0{mpwQmD!d?nD5#rB#*L9sBC6lL-#If^WH_C z<9?szOOe{?Va7AQ+J=9RUmljf&X@fh*ja6o>42_GL#P)m_R~>M5L|A=ow>*)8 zRN^jZ46FOl7m4oktCu>?)ywwhfs{KX=;g(30h8DS2mkNUpD$@+)O8 zT4ak9`@*#`p2-n-6_d_#XcKIY*wQH(Q&x&FnSTaiORxyD`S)gEi`o30k~0BWSq*1+ zvjEvyjXNc01Da$t?V8*K&@3xwhPN4@c~;B4&CN%c9dg^odoF&f{k0AZaXbU;XIzk3 zb=uZO^s`QI_;+taN{QJ-$*K_R8uCUGT4Ex4v$v%xkUnkk(u`B=%#fMyvX8}TU zAC&k@MCM{d=ARLmixHTAMqDmNT>cqhxfo&jm%39ki|fKNzf~QHblj?o+i}eOEAs7* zHiuYabX@6G?ep1&`=Y1EcPw^syiYM5G1Bt4a@MS^!5cA#_{Y@bgi2JPNaw>Yuex*d zwxdmb({IbG^O*sQjOm`HxCV`wKRq`s9X>V8K2D_lF}Bfk9e!cZOj9oQFANOB@B?12 z5}OU|_RwSFN$%E+OuQy6U@MEQf55}0nund|ALi@IR?MDWB-Z1+>#*7T71?a^@7ZPn zhs|GhETJOL6S37`~~Z zV-JX@-0N`cdn9HpK}ou#@RkrbsclYiIR9?p2YgyzhSJug`qyd`u$tRi9;Lff6H?i zSG+wO>&MsVjrYWJ{H@Y```U+NJsLatMDHh>*Fu%B*Z-%n{=LJoKCN@?bKVY3AE%P- zbrqh&{~uy~LSsSh`4;BihidDZ1EKY|cD$%h^D#_qPyxwGB3 zX9ieq(yNj+QbUhtKi2$i_jlB1LA1g{`!2_QTF(Hy&t(jzO2(fryL}?D-b--4^KzK| zy;Zh*n9L`?`dQh&$D;2o!YxdBXn1@7qk)cozq~`_Ctonj|4izDwB+Q}EPr15fQ%Qs z>DFTANO?QdaxMFZj`LO@>j14fQ||qOuocR^Ul9How+nv4eObyHq zu1a}6b*Dbrz%w&#cKRSgpYeKVVqK*AV3#MfEx^TLaKF{D>vKO$V$NbE;oqARBRP7YWLAME*zg|$r>?`RWB?av4*cV0Bx z!ZR(&@ovfuzP=T|*P;@3*!Nr7;d#e>;XaY;!^g&U z-)VC5)2vQwYw*G1@Y?w6tUbQP!WskXI?8mneT`bVv6^9; zC1I={hLv(10uAJy-1)%{DYbE>63?8CnDWUO=zO;5qh;q%gw4)Ci1uu@G1&LEe~NEM zasW$hIMXVYfu1jIEBLFVG>d7B6UEC40kfRWkdzstoISPAJ9}v#sf0+2_~Jam2Ou_PAEA zjl$I(emEPix{{ZSEf20v83F6aMHMT2Wo%;CRr|!h-ejl$>jc)DRGBT3|H!tel0F&z z?_H<=)wTIwvs7tn!j&mHg*)opkAH?W!c9Y1#$9^IwAdRBUq0W1w?x)0`|(_h4r!$s z9nyx>&B~l!H?MwSR`cvGjkB7JF`7*(n@ zpTY3H{Tj|TR_L;OU8%WQK0GGC2z(Z7U873cG%cSqKaOWY#GIe%Rg)jaXP;v&LORw< zTKywid>&kJq-G6e80J;Si=y#;IcnS`{A*`jdXk5yrDmm!Pt8wzIlVbzGyd1b|4jU^ z-#o2LR$;@E?0c}vIgWX35*BGRu3Cb|gFe{~*_Mz17i2N=Nju&UMq&sOP%Ur}WhDDJ={(PtC&# z&B_>$H;ZM}Z*AbEV)vj|^oF^?v5ySng-mb@=h!_G<{+xPQ186}q3v%ZF)(|9ve%y#8BWSt=O+ z?H=`B4%b3H<5&yX*nUr>;<3I3#{Ir+hQD??Ev`x%T@Ncr(>CUa-lU?zGlmp%QG8#JX>(*^h4hi&u9KAp4EW~ zzF=|6)YM^^0=2km-<@u+XH^%EvK{`mKbAI3USxw8*?9x&K^8C(VjO@b@qs_ zFEZO03saozuWVl=YWqDrmbn;l^lf6AtMz+NFRrHv6K@&8q73*dK{w&Qv4Aho{3vohPH` z^+!jW^I!$z8FH^|zTw|!-n6#21Cdyd<9yEb%dy0EIoR|5X963|Cyd75f7fGQ=xA4R z?9X>~p4V3myr%tR=@{Y@y*zQ8i%)!aRovH%WiDM(<>%6Eia&lwjU9M@r&P0uw&MMp z#*X_0hJF2qZH%d(7(IM-fh!|<;U%UT=lkU=AYVbwJdS*BBIUy_PavN0sP;7MoIrYG zWPyS`4Lcv2^x~ScbWFwdwF=W?9qHkyBYo`88?#;WyXpu#z}OXI3Z=fEd`7XqcCt|% zkS|QO``R0n4x!z)m0=x&uWUZ&{EzZ^t>fK>h0Sea6HQXZ(_GgQt3~#zeElL{y1wE8 zL%1d%+cSpWFYvnsUtc(%-~0I8lW#H9cNx~ig^R~i=<#efx>obd_MMF{zkCJcD=1$n z@|7xIY4VjWUm5aM7hlUW=i{rsd}ZNlc0>6gTfQ3OYfFb@>%j0OAkQ8DEN5+ud!<-MeQQJ>rW%d#rOd}Fv9tl{TQWLg*<6}5eutuL zv81Wx-|vV8R~&7tyS8zu@?ydFQuAnp#`2)kBH!gcV~D?|-{_6C?Re8bJzvVY;7}|S zO0DNhTbBx|y7&;Oe2Mf=>&cp*h2bZY4Uux;Q@fersr7OV>xpI7Jy>hL{Tu@i%5;omLy0rLaN$wmpfg)#x$$WO&zhbHzXQhvl+ zPhmTc$LqmFOi#>MPnsMHSPxu;;-vLgGn1f=aZfS-NU181#RWgegvuR0njM!mz?DBl#r@}E+*9i z{e!x#Kw7popvnC8WY82+N6;mt(?Ks&V>n1PMuKMX*D;{WNfSU=4u&e zK7YLzbUo<-&<&)AKuNU022kBLVvQ#(+5#HUR%FkD2IdK68D`4apyf>EIM51G3(!5J z6F~-3ZU;J&Da%8Ul{$kiWXflN1~KJLAT8xq(2mY>v>l+Gq!&QXk^Tkx7fXp9JZJJu zXMnW4nV=uIUn9_uq+>uok&XvFOxlP)%D==plug9T6It!sV3+o(h;EjJX$8G8g+YsRJS*% zYZuY&59&r51UiFs0VsvX9SJ&`x=TT-yA0H~i|F19>PLD2bS~*3P))4|pls@X0aD$s zKqI<{?oXf#N%F?_i%4Fyv8mJzfErP^14woAL6f;(chD4)TpV3OItNsXy2YTz)SU-X z-36elx!-M|*`(V+*O2}W`hm7u2}+^v>maQ|Z-f5M{oVsDCw&B3LHZnY0*{-GmRIZ0 zaF8B%B49O=5k19;B)40(rZN)!qR4Nbi7>Ngsd; znEOva7c-V*VYE?ovq9NvAq!yqI(uts%)GY?7#z0Ub{yGBGm^2FX7t%P;ENV;w z-OZy-12tlvXM&EU5AFh~{ayhLq3#=?p`>>}!$=>1TG0lJ5UN#UF=#P=y&ZHL=`PR` z(mz0tGY9vB_Am#{FuO{1n}gPKzt*6KNo_%okWK+@q;5XwW$I1>Y00O8p5uO(gPtc{ z4SIoe9q3W&E&%gN$x++}$8AX%EOpdJD9Hx+^hWs~Y!% zPUf%cLG4K!L8p+O0^LT9t)LA&+K(XBJpk&?{gTnd7Luxi&Lq_W)#Lfi05xYFF94}- z4^V&Z*B3N^)E{&n>3mQob%%qFr|wpep6TtNE4kkbpsPr`L06Mr1GT1Z3RbP_(b7OS z@z;8wMWhCxn@N8GT~Cc>py@o?K#=MV0sVvfjRLJCjRD1fdF z)R+u;h>juYkUw#yg;Wr1wBylKul4%~ZYsUBFbn z2Hnk6+F?G2rqTiAIYVkg z^l#E)kk-Vdpg~OKF3@1o3efqa`#`zO-(HaBzQ6gx~Y)~dW63=0&O521KLPB9&{5mdV};BMWC(x zbr5J9X(;F!(nX*FttI!PK{c7@WgyjE4tkaQJpg)*v=;O_=@HPK)O{TEB#*lp)QYM2 zjxf(#0Q3VjYJz?w)du}UIudkZuH>KxNRQhaR8k7yJD9Dt51T|pF2SKBmvXKEFWXjb* zKafU)H0AN2yP3)~(BDZjK+8#2f@(13IiQAZq-^s+W0>+n&~T=AE9ghk0g$HbsRK`8 zD%C-oNi{)RNJoGc@GNA4)}AOSHvk>mR;V%Pc&2wO=yg(aP%ql0HE0!S14xhdIOqeW zybbgrX$R;d(o3M{c(hk!KOXH3(CKaEXzzkf<1sz}eMah0*G#!L$kSc+>kmpI4FVaY z3qU)V@<`BoOnD5bGgB@Foth`Qmw-Mc?FDI`KLpihD*Hf3k-i3Hk$wQ}WXcCX?=$6s zBTd~Npf=pEFQ_f4KPZoMK4?32hlBP~cNC}-b2S#!F;8+e5%dY^ZII?_FX$|$@+qhn zX&LQ{=o#v+0liDzPe7{s1?URy_Z{d;(odkPNWX$!q^=KlLqDc&JCN#j z0Nup>x`Gywx`S>e^#Z*>-6GIO)a?)I!jdloX)4P>|DeVLpp~SxpnFM=fVwc1$3Yh{ zmCc}E+Q|6`U=UMNNdY}ZjXI#mN%cTakd6j*W-3iUBbdr@paV>0JV;ZS1lmQ78KB*y z%Rw)bt^svrD)T@inaYizUzy4+pjxaOUw|~duRxzr_b1S&q=TT(NL~z+oX+$DpbME^ zP0&H6mkvs0dLu!a-e}OT)SV2Hdskl1G*A-hub|aT?<&x9Om8meBwBngNK^R`RF@k2 zKu40k24#|d0R5Gz901+LR9fL?i|V!kHRXOCLC2Chftr!d06kCL9-t4YI~}Ag`z%lw z?sqk)D`_sM8|em62gc5=AU)c4P!WIK4eCqU1L{Y53pA8R+Y7pqN6Tnn>Sls2;(m=l zBT2`AMv;yOt>)2Mfo`D1+k&2^t=fYY(DtW+8k{6{UJ24%-4D8wsjLTGMcN3un)DRN ztK|S?GUXkhi(w*`vbG00_fVsLCw3WG94Z4wec?i^y<;}$XEIn^opjVknBhYK4 zV?eKyjt8B>l!t&cFT+6}@z*h+k4Y0i{~=uh>cetO2Tf)<)__!Z9q0h}djfQjv>EgZ z=~>W3>OK#;mAbn@^;nL~Y%`TCP^~lNyd48dBOM2-O*#S8f_ZKOx|+HpL7Icnprg6p zWKcF~8mJNJub}&=dlhI4bvJ-i_i@k(+;1DG6=?^kHR&Z#d+NRdx|X_uMy75G=rrzE z2b52$2kJyR8Z?i(O+f3YTMW|U4g~e$ej`A=Nuxk#lg5GOQ+E<*J$3H~>G@p)8pr+C zgT|9Kf+moj0-Z$NZ$WyrA3<~ZYf@vBgOm)qj#LA*h36;@^d`?y9Z*Lew?3#n>)O$v zk4Q~G=d#vq0EH|GTFdmdfgU9706j!{3ACKYcm=eD$9Th{cR_7jBpa)1lfW9Ej`HPu@d7wQ!+RdO>Nws!&Qcci*Nk@P>F}(ti+MoxhW)G>weL<6D0O(9(kx`cEcXdZPJfF7Xkc98nQ^Pt(>Z#U=~(jL$p(p#WM zsCxjU8lGcub;4hhL3fd=gYG8P0(D{8x_~r)1)wMR>)D_uNqs?2ksOl3Q$CsR29QeC-8yqEh`2fatC33{J&1gMa@ znV_?%+XJM!y+L1dzy6^8q(Pu>NEd**Qg^L1$982&Bg?1~uh=LqW%qMu3`; zE(V=R-SMDa)SV2viMhWNG>vt97f9241=NAM?|?dz-UFRV`j70#^u7S~W_sB2W8426Mj|pz}$WgNBf<0i8wNd7wVjy%DsC<+ufO zDa-K%NYncYG>N)DfhLm2(09Za(O0?$;eOo759@4e1=vboxv& z=uZ0EAW$~*vK*wTtOPBl##+#Ar1hXBq{l!rn93&5GN$q^NOgY%t>S)3&CxoOl0o;A zYJhH}ZW`!8>ec}@V-D(rn$sHNL7Ltq&?f560Bt5+4%$Mx26PkCn+ICQ^frN1cPnTY z_j>`fo3tDBGU+wYBI>>kT2I}y7N%|n=wt5J0Q4VHBhV+LV?hh4+Z^-|bz6guV>$9b zEm)3IKyQ%pLA@DwyMpc`6@oq|^#avpEcq4`vLvX+S>l&TEit-9N(R*=)d1bBmIOV_ z9CQJxZUN}1vqbl7P!_2#r~&Cb&^Oc_0t&DWEdr_TVo-?QXsgFyXALqX?~E&~0?qm2gD z5Q(#~yKnXN`U&>SAEBWNzE6X;sf z8KBHn#DXn4^rJ-pq1S34bZ)$cR;I1AAoM9 z?kAwjshfrJt8O-E1NS=)w2{;T^eE{>&|>Pg16@Jg(IC|w58BTCrh#^l#EW;5t^{31 z-8rDK)RlWts=EpFI`@uHAX@zfm%Qr#h-AGzNs&`+c>pr1(-K~wSH%kPV+dq4hG-8G;}RAXd3=|J=^g;nz{!-s_TI_Wpcmjpn9a5p!%dEK$lWC6EudpMIhBJ2IX+Sp`hbQ zGRo1MBzG|zM=6 zQqBYIr^d~oZ%8t({w?WF&}QafD@ZlAgS@>ZmE9m8X%8rw^cLs8t{R$=x`U1*$q05|mg6dr=3ow}8-KkKbOz~WPyy+0pmUgm zWuPgv+6s$g9K1U7@=s7E^Rf=KgR~LU8UMYWC-E)%UeD9`{(!U{-?bRWN8?-bG9EOJ z$DIZmPnrRmK)Mq29rH2=6r?rgg9bAP3qgH(+*?8aM_MYs@@RK~BxhdFJ)mz$_kkMV zzt^)C-=~ls#`p8I&|{!~lQx6?kMs=a7t(W}6k209s6OdcP&3k7pf;rUK&O#D26ZQW z4*Cak|21e0>3h&4r2m4RBqg1UD>;%M^gO8s=wGDg+neWn7wC0bXbwQ+(mI%_WPr}0Mgve0sS&6z=~&RcttFM_pl4{m)}TD5G8LqG znGU*$8drlxlIDU&k!}F3WGah5+nCC2ptek9FGy4Q5Of(e_JO98z6Q-8{Q#2o$Ll!& zx`QQ3>WG-bR0=?vN)ONt)aVOZK=k#s)jai%gHbURZS1$v38tO02%>p;t?@dRiE zX*1{^(zBo}rt&=K1g5eZG=-^TpK7Mk1oS91T7VuSwFW&-Itg?yQ|SP@j;VA4ZDJ}{ zfiy33Krd3`M$k*7n?buse*>*zD$79gn92&!W~TB9NK^R&^gcDd1ARdH3G^Z9SJ2;D zi*2r8Jw}aj$-3{7Dnv-wlbROtL=IUlp z8`5o{wxl~j(kDXi0W^$OyBBm9bGjOI1@re1s220G0W^iwtdr^c%|U~hUTe@`Qd`jZ zq*FlW@)%P=YQO2AiTw3y&?M4a&}7mLpheVJ1e(cQ-3EGux!McTR6Ybr>3sxxlIb-+-AttwXeTvJ20cgW2zs7$ zI_PeuQUIFERCAeT~g!B>UQ_|<4K0L-( zpdudQJJ8KMMu#qDD*2%QQlmTQ0I4VFAn6>?IZUM()R(CY0^Pz?WVWE@WhJQAIZ_kX zg3?IqLA6Pbfd(>_O`!fvWgBP-Q%UJ+rjiC~OpSV=zmOV$nvi6M-XNyZ3^ahL3hBOhhg1S>d*HZU>km{}hb>@B>L8p_R0Cgcf4H{3~?Vt;(`y)tu zO9wzxxnEK@TuqRYL6?$hfX=3F7m#WcfEM%Do}k-EXM>iI&IR?S#ypVbZvp6^{Pi}_ z8q)2cwWPm;iWskC-qs~2%2{{-6r^>(1!*cjf;Lk_W-4tVC4-(O)c{@2R9b;lqYdaq z{@M}r5~&kt7wHU;Tv=iU3Fw9sg{FhFl(RrzaKEcT`$%&^Uy^PBoyStj92q^@cF;lo zx*PNhX%FaE(p#YOd9=Nt%TJV}eFRElDVrCVsk8#sFOpwR1|3D}2+AUz4!V%36oBS1 zm7bssrqT!0Ks^uiJZT`P8$E9*=x)-5ppQs0N2WF-`gl+)(qzyxEZe1^H%T)0KucK! zDqvm)fx44qoH9nB7|?Rk@t_e*Pwwhn z!}QvMj$nFIL7J=Sp!L+a8uTzp?v*`4x&d@CQ&|L>&s4U9RQGw%Gu%(^J3UK!1GJs= zZ_s_z{ZM4o&FEozKqlxz?$-eH5vdXAW74sp!K@nvAoZCZp#S5qXM?^X^#y%RIuCR! zkG24$`CA04-B)V+?Vxm$-0?Vq^bgRb)VLpX2lKZUbTlpWFsLal{ut;L(k4((*6eMd zm8626R{lVzGUc;Dr;+-C@=51`&S#0HgF;aN^jSa2pNvO;PMQn)f^-AuGUo4fke29e z&`sykyFrUcpMq{C{U2x+kMS+2w2kcdlSRLPR#-wvvtn zCDChTbo3ZfV^C+(v7ncEv=*R`NHYHU9jP72OPh8C)h3+|I+|1fYC$>+bTa82(CMW9 zpdO^bpj9l#aL_}fQJ_aj<3O89lR(=^mx5j*{T1{o>1xp1BpI)KpR@q<3F&6g8??)E zT+gYUSArTBi~ZJu{z6(0YC?JpbQ(*!3Dk|LYy-__Dmy`!(7G>ys?n0KfX0*L>hUSk zyP$)l4?vA*(@#OClD-7J#AAF5dY|+YXg}#!P!cWVI~#oqQV`UDR10)GsSfBQQhiV- zQZ}fNbPQ-Eb1&DWYe}s@8%TMer%0!Oo*{Jty+G;)dWF;j^d{+Skc?(|J?DZxCJh9= zM!O6Hy+e}g$oENOK>s0?g1#V41$|ADtHSR|SAzaangfzm_Fm8Rpk&fbAQ}7ddTs+P zC*1+Mmn7GCt4a5Q9wPk{w1Kn^^e9QL;vOeG1$vUS4YY~06Lc)kmRyq^N7@5Ap7a)| z1!*tn1k%T#)}+rtCz8GfwIzKII*D`ubTUbq zAT6k+C}OPdYN<<=oL~C=v7jG(CZ|*`gxNy z9P}2c1oSp(9H<&=v0U>6NSA`DlQx}edf8S`VSg#*3!pPeyX9BXYoPpG`Soqk8LUn3 zfv#sg_!u;mXXbNI;3V1aE6@becc4wApFzKneg!pV4w8%EzoZ~&7mtw!`hZji^bP4K zkfA;UN+UG|H6$GmYEEhmYDdZgbtZKHok{8hl5t{izx*oKTvcl8EHLe1?f@HD$*vqyUmHj;LM9wWU9dV=&8=qb{Bpv|O@ zLCtu!z5wNrz6LcX{Qzo7Isj@#GEi4?N!37YNHsxuq;ya_QYNT9sR8I3(qBMxNzFjl zky?W0lTHNPKsp&DW4B(# zrKEA7o0#4t&4kO&iB z0BKGi2km8g+d%J;c7Wa|y#!jyW4r>A`(s#12x`G&ybC&kDSrTxyCJw<9cHH78}utv z=@0UpC%+B?C6O)woz0X-f+jKLF`xibE(IOQ^ezGME{tb7s1tL4Ip|K(Y|#6p>p-bI zm+dYvbI<{F9FNu<bzcI-b-Ew2V0@0zJhX^ar(O4hDmAnS%>J?~+D>&f(Gag4BK= zf(n@4K2Uek*Pueu51@5C#sSc79wTWuVhNAo2g$uFucrp+ebRW4raTEWjH%23T|l}V zG@Nt|XbDrE2inM#Zv-9BlwSvFDsO|PQsX1gG}5R4kG=bW>wDh+|NovE?TE!1jYfzy z8qJzD8qGo@ghmLBMy%0j(`YmrAvEGNLTH3`G#Vj<9E}h{2<-^X35}2=d~dJ&`}ElJ za`|4~ze~H|{P}&4%jNs|UeD|Hdj5I6-hcYL4P7FAjqcDa{isp1{D}6|EWe;Q?J8&< zADyLaJv7N9C=A-Y?-80F|Wo^ucK9Hm3IFix>9=%K4)mY5cH_VZiDKj?a*3jN0hJm z5|MugN$3UD!%(v{8@(tUixz1_I?B-wPDb}>2dATA?Vthu^Yw<_)7Tf$`_ek}f%H0B ztoho}xti}ibid|{${Cs^8ue+!B(zbQjJ}bkp^N?3fzHz`Gfju18qIeUTCV-2p)ftO z=b($FR^;Dv8`|zP?>@hWwwFFcW28^eSz4_JovzisG<}2a)M`JVBCR%vw$U0R&gK3| z1<1czA=+OvFGo|PYf!v&BRW{Cm7~M7+Dg-%s8p-nixz9O8nmU>SdA`_qR$&zEe4&S znJ1%kX)2l{9fW3RwIt;I6rVqjXgU&Ir`3){i?mug8mTo-M(0ay$bW6^Xt8Gg5M3;- zM+MU7C_}6DqLZ}RM$`AGLaY6RF4bzj`FviiHAbQfr9YwPq|xYCX)M}9?~Xg6B&iMk zb64n2|E|zo{#~Ie|E|z-y+`z-`}7{M(eypqQ@e`K9eT|qpl3C925ORKqUWR|(0pBG zA@c8_2)&|u4QiFHN3TjZqh{@(0(EK!cc4SG?mfu+xqp!Fe)PFii%!;+uR&|1$55Zt zh<4IFcn&2<`3r`2Uw}sNSxS&^DcV9>hJvK4(FNLlBl7RQ32m>s4vmpsMd8w0=y2`6 z1D&hge~5hAebo6wvqYnPG-48pl_sM}(lm60W=TZnX_lF2gl0J$h3U$VLN7^aC`s2l z2i+;1iq=bg$iMr3-l$I64}C4gqj9?T)6q2P5cIO{ZZi5%Iud;)9gBXJ=Af;$ms8Lxm0D9Da4;JzfLK=zIN?W6+bvMG$JDP7SO4L2r z2~F3P?~1ymXmp~kcOrV&e-F@C{(FETbma%41N`>@tNFI#h?w^)guq4(-5a(?PzeXh&%}ijWRL3pFAi`JZ(KXfM@eXo9pH#YoqoIl7`U zR4~Pxxf%J#wx9!a-WzC|^fpS6K0pgJwhNU_@y3Sd4~>mLhwHrE&=Jy}C`H-_dB4XO z#$>*k=7blyVbebo}=TZB%QE=5;pY!SLwW0#}d^(eF;|7R4fs6Zp$K;G9c zLB6-qCDI4zLfxydMMLX`qpMZNBkyaPAm4828fgNG@jo8upm?ulA^)DU(XBdf0rI}$ z3GyvO71G6MqQ(}YM2%gBu9)hbcNJQw=SwH@&)0<>(%22CR{9dHmcB(3G~WO^Q1kta zmQD5MOI|!QOA2~gBaTDQNORD$(y3@4&60%<)-2`7zk>?Ys`KtfuS%=XYtlm~R%7eX zAsV|D6>A4iq6-i3_7ZmS(0t)&y~d75f0K4YpGXtX-kL8K&Cq<4(UqF70Qq-Sh`!Z` z<)~k}27M>ph>p^8qa2;D=SBCSRM#0j1Xr%P9cb>*}qYE^4gLj^G@D&=P zD@raHnlA;7)7azCPSPB-vvew&t@*N0uI8JEw$Oa%q3twZ6Y}q`8SSUBucA2V4K!JL z7aghjI?)2nw;ly)zR%G1nlEw5(0oZKSz~9T!=$6p;nE4{Sk0G#F4TOb$iJ5|l&jkeNW9zi>3FCmu<%@>N!)7Y_SzBCTyO1q+?HD5IHexo$V zHxZ50eEXts%~y>4`zt|38oL55lWsv*NVlV7G+z~3sQFf*Ej8bRXsqV*T{<*hFuFx! zw?Vf`+o3;8JED&@UnKH=S2f6&hWvX;N7Xv-G_*>Zi|&)oL0uY~i@wv?g(yvXS&WX< zURsfVzBcrP#=eK1ls-gHNuQvPG+z(;R`Y#{j@5i|g+sH%qgOOy25OaNqF1FO(BCvm zD*8dQ6e0f(iqZQz?|Srsv;uWXe@35ZY$fu3UzgvoN5^Xi_o1V7MT5vcpKmFjjcDvB z)F*9&HcDgAdd(MszSn%?(Q%qD8~JC+K_mDM2=AR7UH1-K}u=F%KM0x?etNB_`ujYFd9jW==M2Bg{N7uG##Z&hoDZ)my9-Qz5?XmOCdT_=PgHPN!Ork=|=RS#+IXRGqY+g`p~@^`y;BBenzXLpk+LpG+zkXBHr6U zBJz(-LXXV#j?G4oN=Ku5=>+tZ#%7=(jXe#WqP?7nj@Moqkbk~L^wL~!z86u8v<`XS zD-80zj(Rm;JMw-%p6?Z)lQm!D6+^Q`A@BQ#yo;jurAg=mX$tDoED30&X30hV9ps@N zowo#Skd~q^q$|-zjV(c2YV7qWQ#-f`ouDh~LH_xA(a#$D9r{K35&bItioVu-!B_IJ zR`ZQQ-tV~c9*g|5q@&SiddE&fVbWZ*opcWROta*opY)hlA^+HFw5!frjiRJAXgBF` z^tr}1qCt&)4$aXH{)&##6-5>g%@>8HYV1UGfHVnBlcu0<&6j|F)O<7037YRvl&bkE zJn8vdiH_3PRo*e_0d%zVF#1&U)uVrCz9&$+=6l9FPxFN@ADS-$WohhgXs)y;I#b#Q zeWCf{(65>=8~OK=gYtCVLbOm?guL$*2l*~Vf7jR|U4haxUpw-DmC%8jH1{#@wG!DHc?TSLQgJ=|~J3bMer}_3p zb2VQv^53Bn^tr~aKt0keXoGY+xB&y(FQTT6Z=&R5}{Xl1@MuYcCn-c0Dqup-AoJOtgn~a1MGy%0)+N z-GykCv>5eBg=ma+ung@lU4>rK-MtQdAZ1@Sw7(p*=sfRUEkui@Md)JbQnZ&w6rt(< z*A^<${;oxN+SQF{3+g5Pf~p8(7FleHmxxe-7X!5Dy5@PvF=M6TBZ9k z2Ssb$Q_w`Mdj@((YDWI&S_^tiv%G;Gm)=GV(g)~D&D@2&KOM&JJEA=_^A{*a^L>rH zKkdWc=X?Fo%o*ri%`z8tNb}Hp(tLEGX3j&mYUV{KLNi~2qBP&-=rySr9ijVj4Z2sl z5q&0=qwVzwu0)fi)EkE0Bht_|^S%AeLEB0vBk%jZLB2E46?)vtk^jC_pj}k&M!QO@ zP?Yo#s@3l6&^qnD8~MlfpnY}TMzo*w9g35FLf31o??!%OMq}fVe{2FeSm(_|he(H^ zBN|SC!H)?DZs?*qZCWE|un@Lg^gz56zd0M(dTb5S^*{7Ne6j z-=*kVsmQB(qz90H&x7bT%{OuduL@}ts+2~fpR`6e^8T!0kZ&A1OKa?cPSMPJpnfUF zt6HN9`PZmMk88fws6kqTo{%0#4^Q@HZbYrRFVCR_tr2|F(DN?@b!fyk=sjsW^uDws zTDHG8ODgiOn}$ADJsI^#r=bnfS?ENqn}e2U-Ad#iTZKmEdheXeW)`6YVTbM3K^d=u*v+j{G~wKoeBwq8Mo&nkdaj z*J%fN=uxd(jr?P4&~%-*1|2A^MF&Ywp%RU4LIqm)MO3F*UPfcJ`>67v`J&M=8aoLc zD@{ge(loRr-n$!_$iIUubgJsPC{tR1PLnP|*JuX?XpPpbMgFmMXo1dq0-Y~CjV_R0 zK%+Fa1|dR z*4-WLc7Ue|=yfR;rD%<6^v_2cy`uTnpjK%udR2M~-J>;{&~sYjMKncgyo}-x@Yaa> z^U!?JXuZZxLVuGcqfex1=oihGh{nWwWAl)IFZpPr&RdGUk(QxvrK^!ok6S4ktFcYU zKeic-Sm4d~D%wJN0|iO%qF=RcCkoftu!^Cv;b=RZHy&*-?S{rk6VT5Z8|zh#%|iZl zv(a8UZvmPhEkrTW#b`_Ixe)E7vCB}7?%!4D4Bfx$(Dzaq%AV$3N65;dH9}Fc<{OI+ zlg6RLrCrgrS|b`=qBSO>N3_PiXa~(#jQo2pL8oi%3Ur2a3(AsiN5z`23gv6QRj5|; zJ&3N>P9H{NG;=*#BE{S`v_>qtRP#+mh0=87{gg7ucL=&#Yb2wKwZ?4pu+}I;{+*Vi zn>6AMbhC6fDwpm@%QZ_aTBKRlpw*h?F;t?RK83=yzb15v6o32B8VTrO%{LR(Nr#~| z(oyItt&xTnYmGVRFIuAl`M(0HL~R;zH+n-_h2E4NLN{ubF66(qZgfe$cQ?L7mr5H^ zq4WdVS0e^dlGYtj$zHVXmgpi~+tw&ZR}_YFrLpK4X(u!&?TVrjyw#%73~3^IO>69n z)=N{+MkxXL^opN>wv`Sw9f2ZsEEVl79fuB(PDDvkjxYtL39i zt+oVJYPHMIWm>Hr`5&zgwACW-{;fwL(x+&Y^mlZo=IcYJX}<5!9h&bS=yJ{XD+<+4 zgYV+CC5=KaNZTTx9+@#HT8cn3rSa$ut+6}mmL{NnDXeN}r{O3;yBd$COS_>1r3vU! z|GOD#(_@r{{8yQcj?#GxykpWrbhLCaTBFqp(M`I_W$4AJ-hH{sJEp6=4(+Nd>O=mS z`_Va?by9eR1HH}ekYR?WOK+CwuJA^*(9=xWV!Jt~n_ zplhT*qenG!B`Vjrk@v8fw>S zZ=uhn4)lZcAsVSW{x>v6`W)>h^_n)CzDN71M%+7ey^*M0*S05mTbhX8k@iE6Xtj8B zv#xhK`m3(;VDFf&au$lx6{VnM($T0zIvxe-icUfkq)aqRnv34j8fTm4qtA3K4}C8! zLc!YoC1`u;aug*MqkW}o(1FqoC|SB09W7Ou?l9eh9@4c1uNt~9At+|C_k7t7O_au> zy``N|gLV~#R%+~?s8#o6Z&acCvLD)0_aGiEm!_KzMz84DEVQNWMhepB`xvGla*dGxZ5y@W>UidxY`=?!$4^bUGkt9^iaq%JfdeTuf! z?!Q3c($}Vb(~oF()zF%us|-Umx}tID0ckvXP}&34YmFFmi>`7KYSC5ZBmX-?0czBU zW$0;XIeJFA4(+a4%FtBJatkWdES<V^@^68%DBLB>J=qt^#1bryAQOOWUAOX$;y;Ge@9!%{(4ms+o633pC#Z^s7{g{4O+trCQ_vooIRPD@nP;G- znz13ugR~ZT zznc-{dkQsaY!kX$V_!sXY6mZ)+jMuMRu9b=jqcUhNvK+yj8;k0&@=k1AQ8Q*S7RRX z?Umi=Iy=;T`5KUWoV5^zPt zm7OT6z}vy6Xg6sC+Fkk@jn#;L6r&ydh%V9&enC0fLD0i|#w~@QbEkRpZG)bYwnslo zHORkiEjm1IRyi5S^g&M%MAXmqsD)_h*89qtW&n z8;JHE3_Gdjr~6tKE#=mnzV4TH_8>D@8pz^xTa`J(_tU+8|9rUr1BXiv7LSa*+SN znnW4*6es@hD#BO-ECu8R!6M7P?8hFF^j)3ejPz%hBP|HRuTGMs$)^D@T`T zwUuaV?Y;~7XX!@X?_lxuG&)(@h)$7yK<8_gLA1iZ=e0v)<57;zn}N=iW}@??BhVun zn~Gl7*dpX#w-^=Zyz9{tX$87O`ZHRiv6bjGjcr5zvF)ft=Y5E-k=CPYrO#2V?o}_k zKHj^wsK&i~NsK9(r8oEkX^_67+<0Il5Ep z)+7H;8_-Lt&!HCSMYK+O1!ZWpHgu_03x0fPYzTT+=WT~Nq_OBdX=ikS#zvu=G&UXi z*Udnm>%6(BN1BH=Nb}J`jm<;1XlxboUu88Kd5L$29za`4tI<}{qv$SOWe@VN){DYb zzeC>d*zvatpdF-N(dk+(xPhN2YPAI9ADf80-?rn=T%f(B*=Qf>Sahn!rlX}ATY~&! zOVJ@Z?-rCKtwb}WyHK`vT8-|S>aEd<{A0UNn$FvRj+4Gb$4lR$vo&@AW$8KeGg@|l zH%sypL$joyc^YvXI$N59&XG>_q*=1iT+K2MRcSBhp>3ym^EDy=uA0%M8v80Ll-@u~ zrFYT2x*MISOM3}_a%gM>x=!cqhDxP9(e=_k=uqt?4&A67WF!B&Iq0@ayuB<$-fuAS zo`@=?OHqn;UxoZ5s?q(bt5J=#20b7>j%I1KMpUNN`jCHYKYIKUZ;hW(gEZnPUPsba zXr9J~qBFE^67r8tMlb8UqtPqUai~=~37w*`nJ7bJ=c1L`!P%%#cWplUlkR;x^3U9X zKG7`eQMdFd`c(QmdRF(o4|V977~MEDHU@pG^CqKyX)5|oItZ=Su9DCV+EqUCuUmkE zF7@ulGUWX(D1X}r8Yx|enzU{idQa;%BmdYIG)CvWfx@M?(OBsNbhXBIp&E@1e|l(a z1lmLA?S`VIJ<*=hKBz=vdn~nVI=AdahZy`#M7NP0VrRa5yEkd7b>~eIN9))XB zk{*Q{(ch(Vbc!C2&}W9$2ty}nzHul+8jntv_CRe~BL?+ojY;Tmt+79vshOvtUMUfs zsx|77|2i7b{hIGNR3p8J9*|x^d+Kp(LkW7^-bR;ejrY+7n)zciLNj-x^P~-^QTiJF zL;4Qwn&3UYKcPfv#It-Ir8TxfUD7tFPa1=MmByiMbd|fJouxg|L}?#1Rmy7`y0(0@ zQP;5)eIqSH-%3}b0^Pq-RH?Bm(0Eg67c{=myYi)I7ik&VRk|8|@81jBTK9PcI>WygbfSMRsL#I_ulo0b)=Ceeektks zp^s(B=xD7m8yzDZjgFO0K=Oj?5e zDh(q4efIs8kLQ|sBsxd^gq4s|2{*dDY(=WRqcN#CKHrJqo+*7d!_-PO8r z$bXgb=wY2V9o0!Q&>CqLx?f`pkbkv8)U3K3dB5Y%*G%ZI(v4_|c3O_^&`wvPUGym2 ziT2c^a4&jGszIr`_p8zUQe2De4{g-U)6qB54D_uu3)T3q&Bwbk|8MBO-A4?G*TW|I zG94f9IL8bNn0U_)6#ajX|4&d;XeDhkYkFKFo2q<0~B>-qJm0o-#-1H)m*RWI zIqp8^1HCrg*Y&B+{J%Q$ggSdz;@EwSjB<`2?(E0igZm!kzU~T~vD-U)$aVIx(D9>= zTOGS=o$1WuuJwzvw_eBnj$a+Q>3$|TkHuuir#pL&+jP3mH6IA=zZ0FYzc_n4%9$tI zv3q=1=lt`WeeUQSU*jA<-8sI&ag6PT@7X(cAGC5a=2soNL+789N-iACF?4 zJ-h3cJ9~5QZMJi-+~?15-izFG?sVo)cO30p%eKyOcW8Ip^FE;P?a1TJG34 zoUxhC*r%Q2DULsK?Cv?)*|WP(_r1^07P2(-65oweL+`NbLQey;MQb1m+@J;^!0ddE%wW7POf zK0R`iH#qx0%JI9-bMh-^t)rZM-ss%JJ)HCZ;k@^r;+)g%yq`Se%sJ7S)4lesH*M25 z*70!egZmu&BmALrjn{3nX>-?xZt{-Kwg1)m_%)pUxc9;R8o_<6S3C0`>O6MtarfiY z66gG5oyRMX>zs2Mo%!9pC2zW1pS!noXRLdEpwHRd*Wg{w^I?i}KOb?PKYt4_f1vk} z_0Im??+e}S9Cx3q?w)^f9y|9Mk8{TE*`XKHcX=p!X>E_+1&Ry>1KVHCW-CbB;6C^|{VD?tOEg zZ|-aA8t1j|ex84;vrpIV*T2=y`%E29Cz ze4R7@8P0p=!_M*dobyk1T%Q*X z&x4$Oy6-u+JLg>Ioa27aZSlBG9|#|EK3Q?^XP}RLk30L>!kPbeXV30AuJ3Wix@!gc zn#=v()Z@-|xnl!;ZMfRm+n*e}ug5?R^to}MkJs+kAnx9_ZspgL5B(9cMe&^;hTl?0!yrwljZ&WB0gg z_qv{U&UYX8o1Ek8o!5x_z1-~s+$;BO>GI>{Zs%i@AP)#+Zp>tHz(fg_yrc88^Y8L8L%{Jwr%T#mGN2+>o)gDtX`cAbE^{Muo2GEFE z-t%e@ZB+H~I`u0U1*wLhZ&X7~VQ92!IORHM+gd?SoMv1^J!`y_kEV$o>T zI8!{Dq?&-XQ%y8^KZ}~8nvBM%rkK3%oX%2BLt|CbO&Q4h7a;i?8qkiaStjot>MYe9 zG)^_wl!q=-%||<{df)T$s}NnOT7-60EjE>)yH!ik1l2NAIeJUA0>!9SnyS#-s?}(s zYK^HDy{}q__ExPoHJ}ew8__!All&_kKl2p@7 z>1eTP2AZjwY05(0zu_6=%SMN)=9qHPC8~L7mTJDK02QhhqGZ(~Q!%-aVT9i-sF8QyGk_?%~4GQ?XpHJ0%2D+`?ei-bja3an=cH`-*=SGI9JEk1*OZ4QsOF;!RSQgoXrgKnx=6LyRD$+VEk*gN zWu|g8NwoqkQmr&qq5V{=(PGsaQ!SdTT89c$>rD-4nrb7uM77D(jE+!kL4~TVrZ(jL z+sQ$`cC=Kr!_dU8kCau2oGorJxn6si;&n z&6JK-sb-)XR5MLk=y}y_RHm9^%0=&}=AoNZ^GyZl3)MnYu3BU&M!%?*pj%Z-O=W2N zBfQ7A995`Rm?}|>Y8ARuwc1pJPF1Z%cdOQ!>d^(N4d@=#MpF}7sM?IGRa;E0=wj72 zbiZo5sRON0?L-f#cA2_8srI0URC`T*Xsv2LTCF-@8br^l`nKewi)yeb1huP%qDNK3 zOyTG|)d=*0YNRO&otEO==V;Wd8e@t@XQ;-ZzpBQY5>Sq6B5F}hG9{y8)fDuyYN{y> zRjH<d^kG_2@m-22&$SRc%5as5YBg&>5<&=tI>u zQ#;C2?LZ%^cAC0Sk!m+uui9hkMJrVM&?l<>rUA4{br5~3>f35a!RQIq5cIifs3{D+ zrW%ems79C~(Pyer=(e|p|_k{h5Ml)1n&=}QNQye;4H6DelCYTb@$*M^x zLN(cxf)=QzqMcOJOzG%S)eN+=YNja*Emh4%<5hD^x#()uJhY2yzNrA+rCNw~RV^|V zqq|j0P?Tz^sSMq#T8?&8tuR%hRjO5J57lZ@4SHC$7DcPpnd;G7stqVcwb9gseo$>j z6IEMGt>{NcEl@ohVkd%hZjwR_#Iis`i@tP@HN%+D~=BG>A@8^^M{@ z)nHQyI$1RoO;!yvg`-ndBhdb;k)|kgnrbwPSB){nqO(-v&;hFPrUZ16Y9gAZnq*2w zm#e0r>8h!wG*qmbj%KK4m@?6ws#z#WHQSVf9#PFjGgb3U`KVsC0L@Y@G!>zzREyDJ zswJjU^qOiJIzqMFRDnKHtwgg`t4!7CYt_7pe}TlU03x8d5O2 zQZ)pfsv2qvL$|7iqtjF)Op&NYH42@f8f}U}W&@ z7Mn`Y4XUMRp=z0_9F?h7po>&1O;zYt)oQdzwZ>G7?o+Kp7pvBr8qi~^jcAE#lc^c~ zRka0Os@iI5L+_}zqot}HrcU&kY8Sd(wcFH#zE|x<%T)VJ{btZ zja3as#i}8uP_(;h7`jR|+!TRgR3lM|YLqD&C8)-rYgJ=SacH(`Ji0+O!IX&3QB6WO zswSIK&;_cgs7y7@l#UjvW}p?SnWijsk!m)&Nj1lmix#Qop_^6nO$F#;)k5@V)gn_d zx=ytORj8Jl%FxZKd_;r4d^b_MpF}dO|==_ zt=eL0MIWiQp?g)^O&zFDwG*vU?J{+v5ozA@ss}xw+H2}Vd#d)MTGav5Aeye~{fKw9 zYOpB;9i$qH9##!Ag`;HE2((5u(iDXjs79l;sxhWm^p0vAdR8^wlz_td!8HHI5o%IR zG9{xOR8!D%s;Q$E z7NeI`OH8F`f@&FBr&?~RKnJN-qL)>xOx5T})f&{QT5GC9$Ent%*Hjx!jcA@~6KYd! zHnpH7s;%fv)izT*x?Z&dwX1fTy3jqU-RN!A9#bz`quPhwRqZzopa#`J^q#73n;`|G zR@D&nfoiBJ40Wl7qYqUhOp)jp)hP6_YP2Z^P38}D^YdTyiE5lF9-XS1fUcRO=btGF zovoUTx-~Y%l!|gx)6l1?>81>Ho@yrgTs6y-jTWlrpbe_IraZJn!`c}2ZREvI8twZ0b)|(p8HvD0B{!Iz= zgKCqh8AYkKpdVFRO>HPnwH^JW+F|NMNvd5a=pb*6Zc`6htlEo$Rr^f+=rYv-G*WfY z9pjYp$Z6HJL{ zwQ3UDPBqz-f}T}PMPpRcOzEgqH3RLanrX^HU#MoI2-O@@F8W3_4~Mtdlib}Rl`i-C`dH|9iSR%ibA7RqtSt?F{W6Qs2Yd5CwXhcn-WmZ ziQch9^r>o+DH-*trl7f+In|Vg7W2z{{5wkMOw|liCMrq|8bA|H^{#giEmie}4JjBMt{Q?aR}D3Vp)A#Kv`jU^6p0qAMxo`Z(WV%5hiWXk zN;S?Dk5;KBpsQ6AO-bl})nqhnUvKv*rc`uTrgtn2RcdUyDFYp$nu+dE%`#=9*{V6{ zPSspf9y&@jAKj%|U@AmMs}`Xu)nZc#I##t5-K|Ues$r;BHQW?|&QXm-t5u^+(db;&7_>$; z))a@9s>Y*w)dW)_TA`YR)~Y6(Qqaw+spxUlG*dddLp1|Ep_*yRLTgmB(Nn59rd;%# zY94x8HQ!W#-c>C`&#D%giqTi9CFnWTQd1ckeVX@5DMv4;R+uVLoN5(%QMKAsgN{?J zMK7t=nd(u#Y6DuQ+GuJ*m8#9?Rn-<#D|%YB4ZWt?Zt6fUs&=B+Rl7{x=q1%2^oDA$ zsSmAF?MH8`4wwefud2T7`HECE*c5`cI^DZhq3B)JFjF`hr5b_WQ;jr5p)l2G^nq%O zDHcssjYA)*#+wq*>8gq7W7Q;6GFqdWg4V01n$l2*YC8Hy})nfE_)e=)FidQW|U#gayDp0a&CHh*m%2bW!sMerw zRclRkXt8QN>Q`+rHKNN@o6vWv&88Mqq}qypP;E1{qvfg{=ttE~Qx_^%?M6ST_LzFn z1FC)KXVrew0D4Mw5dEs^8#AO}^s;IQ8j+~ypD7G|tQwAjR3l80XrpQr8mSs>ia{f? zy!m3$R;qEPc(kKx0ve^7Xi7qpRFl!xswt*abf{_?+D0|ql!1;{%|xSBvrO42M>Pj+ zr+zM3N%i&(o}`ksaB(%RclPO zs6(|5jaRKVHJ~q58_}++O{Qk_t7;3{O|{k3hDOiz9;0@&hiZqZ6YZ(m>M)PoLE z?L`w*`%L{PO?3cGR2?+=!hOCh)nK%bYKSQmEmjRflT^b^5vWu(678oNWr{|3tHz+o zshLsW}Q#b}CZ37V-|YAQpsRm;&V)e2K3I$gC29j01sszK+g z)}kX+>rC~iShWGoR&6vjq1#lO(b1|crdITfY8yI6wcXT#{;JxEj#ce4b)y#59+a)x zYwAN^oaNogew42|U>fw!%l4}G7c3O32Ae|AN2;Od7S%9QI68fvH#P#TQjIi4p$^q( zv{p676pM~O+Z!8)UQ~@YC7@Q-MD(_5k|`M-evUUb1%0ZTYDz;*s_E!E)eKW6Ixxo@ zn}tTs@RV)JLG`M+Xsl|UDIe{7t~a6p?WtO5Dnd1?#b__p5>qL9P_+!jsFs^5&?Bmq zXm8ajQ#E>7wFbqi)|%?jr>ga6U)2UvBiiOXZ_iCAPPN(8f(}t_Mff2#R!D#FG-kw9y4AoFm7#gD*jt*9h zFh!!Vs!=FOHQE$|B2{D2p{jAFcoeIefRa@cO-U$GH5nbQnqo>tsj6uxMK#@&flgP= zL`SM-nX*xVY7R~Wp$k-#O)2PC)l{@lHO-WcMlbLlg$#6& zYNja*MXP3`eAOINE;>{-4=q;BHx-~v)k0LDT4X9lOH@nHC90*SGE}BojtW&POqFQ0 zY85I{tv1!5*HmlK{i=1QdURI4cfAd$Mzzt@gyyL>qX$)6Os%LuwGGv(wwpRolWHgW zi)xpt8?9IEK@Y3;n)=YlMc%IZ(HhkO(;zxP)fd6{xK)EqA!xR0C|au;W(r61R3p&i zs*$EBv|KeBJ)s(7ibZd##-XQF<4p-@uf^V;6VcPENv32pUo{0itD0&`L-(nsqvupJ zOqu9O)hzUaYPKl{wW;Q!7gh62`KVX50KKGIXevS>7khgyM(b2dOrP_)#!ND8uYqqt*H*3ty+)XP;D?Zq9v+L=q=S|QwzFIwH3Xs+Gc7;cdK@w zcU3!0UFb2@ZuFjNkEz!?PqhzysM>EDKtHJtqV=l2aYG75V+*`H8G`<%8fpqdJF14G zZq*1=B-%wa3Vo&;ZHhrjs@|j#EuQU#KRUl2E>CGU`=LF{PqEtEQo^RMSlv z=n2(K^o?qkDI0yJnuGdPb4_`kRP)hyss*M(^u1~k`a!kWRDym{Ek!@7mYK>?_!93v zSD>F&D@|2sf7NRAi)xLj79FTshkjM9H#ML{)kZYpU~i2kQ!|>S+Jb^qTTN}~MAddQ zQnka>iOyE-LR+bJn|jb<)m}78wa?U#u2LO9TdNM5d^_<7ss^Lcsv)LOv{p3?g{g*{ zBGA*Sk!U;BC{r|gRy78VQH?dlp%+!-(OA_4QzCj>H3{vgnruozpQ@&!omJCJ=_u$D z@7`yiNYzYJ7TQ`h8;w`ZG3BCdRP)fTs`;h@w7qH}+FiBCRE#F5mY`_WQd1e4p<0gi zQmrsmqGMF6P>gD|sRo^;T8s8ptuxi5C8`Z*AJs-v6S_vV8O5r$m|9VVY8#rQ+HUGV zt5iGDeyUxjZnRdl2TfM(HT9tu)qXTZb-*-;)~ot<=KC9}!KM(jQ8g4zQw=kPqY;;S zk8cE;t{Q2ILfffEqk~jqOtEM;)i^XmHQtnfrm7~QLsXMY$!Lyh3Yw{!YDz<^RMXKc z)eKW6`dT#$9j2OX%0W99dRLT-W~=6z^3hSM1?c%W@4P}&5jsJ&7`>ocVk$+asFtB- z)pAn>I#ab0y{KAcsz%wWHRyHKT2mdmN3|ZcsWzAz(Q4Hul&&jkHnpI&s;y{_YMZGY zJ+9h;PE_qQb)hFzyU|IiJ*HmtlxiQ!Q0+Gjpr=&_(aEa5$RP!zXH`ScDXO8SF!Y>i zI674|!W4;KP>n*Fs?nwx^rC7kI!!gs6pvm~O+crsCYqAaI@M%!rfP~Q6@93hhR#w= zH)WttRWnhxYL+P*eXg2=&Q{Ge<)L2He3YYFU@AldszvBL)nZc#3Sa6yPfJm*YMH4V z?XOya@>DBLRp<=WYP3+b##D>WQLRH4s@9tt(7CFOC||Y7)Qm1rZ9$7wTTN}~a@BTJ zpxR;TL^rB-p+ePeQx96L+KZN|_L=(8TGat`nd+d)H=bv^YA`BN4KanH=T*Z{v1+&} z0)4IO~ERVya_4j%=={w+?#}7@x6Jcf#{QN) z1X|af^Y?O~wKn(oZ`n_vwSLQ-fyNH!H5F(-|F0c)@0Dx!J`Bf!9*aQ7hcoAIIVaG% z?lrpSxNEt`e=q;h^Z#D^{7;_8ee8yFKmT`*{~Kn39@F3J{sfx;k3K$}{SRjk|E}ki zySL$7%W%$d=M3cG%=1T|6KMWG$KA)sb)fUzYj@9a?Vj^n=5*)zcRA2?1v>8Tb2#U? z=eyqAvBNn(&^dvQ2YT!Ro$u~{a~)_ccb-3DU3dQf+5d6vfA{(S`{Vono+tmiKmYB| zf&ab6|MwjJZ+{N|zp($|yiSL6j(e{*_k4Hk=6X15{a)t`XK%wl(8KzrWY*1Uq=$t^u-T7S)=lnqD1Ul}{@A}_${=aKo_u5?t+E1Y4?pW8uIX}=jfsVWL zyB^N@fzAnZ+@0U`zw7*e*ShYtyAHIUK*!y&u7`7epmPEpcjtFKobv;n6X>`*zw3Y3 z`TwqU-D`I}oR59(`L5lMQ||FV-rVy8&Ep>bBj$AH|DXMznEaOe{O=n3TOPaNyeAL#r*$N!b{xcAfDkGlug?m2<%p6}W{KhSaa9M|sot^=Lp9uMR|^8`A7 zbC3UC^Kb4vfzJP2c7xq7#YaGrwfu1M-e;qrV>-sI{3}>u+&jX$B z9(V1I9gf|xt^p)}Ob6f{He>lg7bIx$qa?c6m-)pVkYo0*+40Jrux`EFB-|_$Z zJ?f9RCx7(g;=T{?SGd`K?z(&1QT}E^$L=|7X@jnJO?XB7Ua z98{*tU)}B7ferDmT$u9FO{)CGvA%gzy;^80LglK(DDwcXmY7P>t*T{c3$0sjszCRv zR-zrXZk4GTJ)l~H&epoMraJVHYCSql>o%Ag(Q4Hu^p@^Mv#ABOs_Z^V#-Q9`mea|qf>V?#}0 z=s?wQ6smP2Opz!>H45#jvC*a&bfju5+D2pJOz|i~H33CwY@#U%ouZnIw$<1aQ!2_) zO+&kBY`Q4}ou`_KMr&-ADH|22=Ahj*HrJGgE>X=#Kj>Z+mmIm z?Wc9iOy%f4)e5wuYNe?PeXClH_SVcbrdsr!Y8}egbF$vlfPPSIL@C;Hlc^bPd6@V3 zwxIcX?YElRP>5sYaqk?JCL?jc!+sL2qkUv8Fh5hiW`}LSqw5iKsy} z3B9SY$)*(aq-v@sjZHJ9qt{e3&|4atY05%vs@dou-HjYmE_zcn4<+bs>Qt>oJ0^HD*O}_kN2(2I zs%oRD33aJ9qo;JQT1>6zd(}47u6xyP>Ocdkov1-$yG-3^7bE+u~Rj8(;c#X|4WugaFv(QY9%{JwrTGd>1wVsoC zrhN1l)dG~SXI-JG2-T?;qhLLAN=&8b5!EuZK(*Xdf$CK&(NyiZ%2bVBQmsLUYR|Q% zIr)ifR)&Kr=U+TF?iot!S2JZZox`4^=zRHJZ88)P+7)?MAtpxyRIt zzEEks7bHE zEK@exPc;YaubFdAd1$g~KANIuU4g04lWGzAShd(xf~KmLqWyKfWu|hJrCQ-hd#*H9 zp+ePabcOa@W2!}$sn((GwC8$L11eH&L?>(JCQ~!IRkZ~bYUWl`8>&!kN1>Xz!_rs(laGo_<{ zsAiyNRWnUlXhDkiI?6^>dba17a?u5kgO((L<`fE%_L)8f*$d zt5ri$nVvaerf~FvY6M!NXHKLk3caWrjV5SUF{W7bl4=~F7b#3{xh0T{R26qASWa<)F`0bI~WdqC8VR>QOB~vvfs;rXuu( zYB4%QdoD4RqF&W9?>yCVQw91;wGyq&H~O1))nn>KyQ=n~7LDyU4WMbN zgQ!bmeOnDF7)@6VK{Itlp{6i&kZL%Zp*=^KB0Z@_p&r#}Qw%ypH5R?7UB#K=(Q&E? z=tJ!)(UgQvP)$Zj+Et1v70ppiLoaFObW;Y(R?S2oYvwFdHac512mMuJb4__@k!n8r zNMj33h3I0{BJ`TZ7Mn`Ya@A7wnZ}lx%F)%T73fustu$4kTUD#kry5&hsznv5btqYP zqu$hjZc}aaj%ns5Q!{!{wFRA{nOjY5s8+Qd{i64}4pS%ki)t4-R5N#*deAefy=b0h z?lbkHCe;BnsF?>%z7YN$0o7o1m}U+!g`&4q!%&W94mU-hw^bw2ubMf^6ph|hjX|UJ zoQyTaq4!kd(JZZ-U`j-PS4~1^Yu#j13i?tt75%Jr(@g2;Yt;<2t=7#nWuc9#+2}7? zH^-EVwm8yz&&fltXx)5M0SZ7nzFDmZ~M_a?M<7DnsK`%hBDMxx!S5c2=!I ztr}ZxszDP~Yf-nx)|u+jKB^5UQ6EDaO-<+s)n+tTGq;#p(MhUp=t|ApZt6fMt9GKl zXyz_cH#$$X2i>Sgq1V)ha#j1$3%XYWra^SRs&5n@Vf7dVn?g{YYA717k7Z$|aCD(+ z1PaowB27`~D%EHdp;oYgH4`>DpD2DH)Zjrl9XsQ%z~;2Gw+Q zrtWiwDHGkLnuUsWpR-Ll=x)_q^sufd&yqL9OtlQH z(4Nap6{tbA678b>uoSKq8`;Iv`E+6Y-&MYsJ5bu zG`7vuj(%0`K!4WQPE!{eag_H8?MCBty*;L06r|dRCTY+8rUA6Q>L6OKvA#bIDHw&T zhMDIUeECZJM{O*AE;LsgSe zy~d`PQc<#M8oFL%(@h!Zc-2g_R%5eF*(hB#2Tjx1TvHypP&FStug9psRESDdi_rUe zjEYSq=mynNG(+o_naWX_Y6Ut_&&f(t6}m~a8hxr-W2!~vs&(ijJ!9)l4d__F>OoJc_M-KweWrfYs5*eswdX;T zZ)-kws|KUhsv)LO)TA1QdR4QIeAJL`QT))a@{SB*#K>UtAQ ziKtUG31#Vul1(Y-Bh^$iQ|qRg($S!52Flc4k3gPLI@clgwVFzgb+dq zA%qZ`5gJ0s2r(feG{iK7h7e-%mzn3dzI!>>^_L6NAM>S*` zM!Twxpuee(n#Ry>s^e&H9rJ`~5-n1lLieann`Y1+s;j-eZLiN;M6Xijw!?V~MEnWoW72Zpci8MME)JZqXm zE33|9 znjqC$bdc7qGu5LLR2$HtTDQ^EgicazMy?L-G_zg?zo zbb)FQI!ycRHT9vzs{N>5UxNdtLDZx=gznbY@33hEU9CEbGIY#irg7A)I)RpI%af)l zbcgCR>d=;FOta`t)j4#7)}1#c{DJQ!s)^`vt(#;@Mvtndpe!9#swoXUrkakjbtW@R zndk}CEOe9hn{CQLFRA9DUhOx}l#gCkEkLbWx6o9C-cc<^f6=-nrc(5-Y8kpw>z11; z(7#nH(Gyy?%2bWMQmsK-=d@D!^=NBd-Ud@6`c}0G{X@0c)PlZKZAB%^e*T); z(GRK}=n>UUQy2PCwHtk<+GFZP|55EjYb|^H*EE2BRvkn)>ud~}hS47m3g53r(33hF zqoy&GqB@R#(7F?*Nwk6L6iU+9!nA1yWvI@gpSA9sX&!B_ny@NIpmh^XNoWVvWORbA zg%ndN+DSDHmFYaEn=;VOs+s6})htss+Eq0NouD(BYsy2rspg}nR0~XnXpw3Wn$smJ zHkF_<)l#&IE>W4O938D%fhu%~Dos_WT(ue(FiSkmulVc#J*)p zM9--vp(R>3*_49*rkaY*)VgV=bo7R52CCP(nWilCp=vfdOY7#Ca?wYsd1$HD%{LXG zuT%?Bjn*wP6{D|JOHiHGEj5*)UsTJ{=~}nKREd67twM{nZndcf{r=$aouL+0Yu!3i zJz7Pz0bQtd8%<4UJ=JD(hSqH{wW9S^+fc2(j@nHfXlvC@)TZx9U8Zidt!fYI(SCbP zeP}n;esqKOJ75|_i&Tfu!`kn#X#^dlI*MM^e#cDXs6=%FCG9uKl#K3GO+f_o5EHYwAIls`jE?Rr^f+s8MwQ4R06D)u3qz zU7-Kd(1a#hnz>8MpT11(a`G-aV%RI|~3syU`y)TWw;9@H%--&BBZS1m+6 zszs(^)UH~Bo>DC}m7%*-%hB_y6{bqmp<0DrR;@PGpnFwo(XeWrsUCHzHlTM@8%<5< z0o7)7#CGA@Z!xu^4^`XHYO3v~4wQXJSfdkVYRg@wZnTYR4?0U_!fNI*~`%q_0v#3mU4(+Hd z&zllb`ORE45p`+HNv33UnraGqdHb;CR8tzNR!v7cXv-Og`! zcjbmH=b7?RgK7a9(UuEMMd(u1V$`oKmzYY?jjCnn&fKu&a#ICrRjovCY0FinYIKWg z4NBLRYfW|NLDhP+ZEo0dgQ*ccq}qhe*Or@2E$9)|*07wm+-7P=XxYjVSuJ562a z1=Vgepe^^9deMujedu9rx!*K^-c}t%&ADOAL#AQ$j_L?{QCl80jiGl{$I%@N!@3iu zN%WcO6naLtjcL;i`a*RUWoyfGrg`*l)r2*ZetfDI2Yc@MgY@&Lz*LAc%nZR$aJs=er*9YXCh^`l)>2hgQD&x58Rl&?CB zR@J#0F^!_#Rmafzs^g{!RG>PEj@&Nncgi%4DpY4stLm(24xO($k1oXwSFJ+#Y!_;^sRli%T8o;r-#Sx0 z>Qik%&uhPprY7`^YBRc5TW&G6qBm6A&{f)UyQu@crP_&J(3ZPQ-RN!A9(4VV;h1|( zedrz4ew40T&VXqUy{kHe8dQf(Bj|nAQFQ2bVasEtarC9?1S-`rPnxFCwCXgvS;stM znnmBJ&Y{D!<#|)W+WZ!&nure9mXl1$DB-YhEGg&~Z8_DHhL%@NN9EdbhA9)Ru9}68 z(w4JLIVe>%7u~Kc=b7@+TB-%;SZ%q`RD?EFEk;Lc%O$2#w7F^-x?k&-n<~%_s+H(! zty^WPMmwq2pc<`PYpO$gtJb4GYTX7?BidKB2_2$!n@uffiE1l4O0~_@j_y$HK&AS5 z)M@HMkE?d03$_o}M~|r&^{V!v-|0;Dn+DL6s)OhfZF$Hvj0ROlP_^o)X$-xsI*zX2 zKJ0hGG>P6(okGXx5>1!{V#h8C)}qYqR&Or2brPMTEl-)I(K)I!=qhb_)-;D|Rp-%~+H%4=zbO%& zubPB5(Dj>aN6T^k7(T%Q!9E+wG9nv-F8z4`kQJeI!?FAE>k!9LbV6osx9}L`q1~P z{pdo~0n;E#Jv@B%457QU-(k}TT1#~lZK$&`W*SFnsuO54T?><@DYUNYG#XW%G0mcM z)j4#azPjg43F~rOQ%yu&s!66~l%bk}j@Q{pHKn0FRMXKCUET~+Ci?n_aCx&(gVxP9 z<)B-R3^f;>lN*$0%13Rg1?b@&LM=2Eq1#o9(G|M9C8ko;u3Cmx+bOJDZmK|csaB#o z)hbgp>QJpggQ~TrI&`mUJz6d=Y`MYIh&ol9(59-*rWW*oYAf1PwawIyx>P&R-*v7! zOOS*C0>pqhi8QOz~wp}(r;qZd^ROoeDrwFnKV7Mn`YE2^dFE!8qpd00-h0==hN zX{ti6t5&0N)f!VR8dj}CpQzTG8qk}njc7`>$<&NSR9nzL^mWu~YD0flZAXiA%jqz6 zqD{-fZM+NZqb+xvdeG*oz35tPxzE&(won~F7w9W=&@_a$QXNJs?G*MgVj4wTtB#=> z)p64V+E#TEy`VZ}nnnv%XV9;zv!*$;z3M#LC@<_cymiYmC88ZvlhE#}$)*&vlWHn@ zOXn)hl#X^*%|M^1W}33luBzGSd(|9MF4|2s4=tx#d%mdvEmAE+tEv{6iqRgbC1`!s zQd1e)OSK$rrCMRCM0=}Np`BE#O*Lp=)mpTNYMrSb?Wfv+_E&8*HK7Akn^Bi;IW4AE zbf9V*dQ7$5)PW9G?L>X5U8Zhyh-wddLABS^hYnNiN3W<3mN$lY1KSaJ~~yk0DY%gXevUds}`f5R7*^y z=nT~|^owe_sREs)T8R?$ouSH9jm}oBK`W@%n(ENGs`cn@)do`|I#0C;b*eU-TF?cm zt>_`uHd8xVtlEM8tlDYnLQ7P;(G#jYre3sEwGZ{F_L~OK#j1no8Py@vFuFu_1of+q zn#Ry&s^jQ+)d|xix?FV%4X93=X3&+YvuJaDC!90QqpMUCHsCw8YN9C#U8981>Hy=o@fRyE6%jc!oQK?_xLO?l`h)qJ$QYJsT`-K<)Kc2F%gm7rTy zOVLiMWu|g;n`#BxS+&wsh3-(TMpvlTm}=3Ts&%MIwcgZ#?pAF?SF1Len$bO~EvQ+w z)zpUWQ*B4rsdkt;(fz7js71Bg)Po*W?L{}L_L=(8L#hL)RdvubgdR~HMz^Srm`2f` zRL4-8>bPkFJ*qm1hE=Cb)95kP8T6*=tZ5ECp*oL7R1-G*O^N6)s!8bYs>!Al^pt8U z8dXg*rK6`+GthggnWilCtZFtIQ_V5uqUTie&_)U4PB@2&+VoTbckvvs?&FsE>kx;Otl9k>GJlP`q1I3{pf7f0n;ElQgsLo zs1BP(&{3+R=)bCCrg3zP>IB+YXJgVdg^pF7MvGKuOta{C)j2exb2V>D$lxceY9jhn zHOZ8WPEt)lKd7df($LAO>1cWV7L;MiM5m}`p%m3@Qw};+H5YB5nrF&Kr>hpAY}GI$O089iUoesz&Fk)}V)U%c(Whq4QMh(c`KOrbcvu zY7=@|wb|5y7OS?R0o68BJ6fXJfnHVZG>K)gDtXx>&Ujy{p=98bFt*4x$fL zhfKrhGSw0EFV#`g7`j|_9DSiWVVXo&s!pM?9m3b!v}p!ir8QJpTRiQgotI_?cHKtm0w`v`FShe2NfbLOk zM31UAnVQjksx7Ejwbj&y?pJL`PpfvAI?;owUFccWZc`6>NVOL|r`l)gM~|oupchmJ zO+)BUs>A3-)e+MudQ^1`W$8QNxM>1CraFnXRGl(SqbF2nP`2ueoCZQbFWK#-yS~V4Ir@aT7|Artv1!5*HmlKwW@Wddh|Ed26Vk@ zqp1nKq1udYP;D`_qPJAr&`qlCrVjMBYA3o`wae6v-cjvAx2pD<`p~Cx#$9Yugf## zqrFrM(5dirW18Ax0AbMGK$TW;DRvkgh>o>Pi(-^u$bsTM`I$@ebm#I#nQk{)y(+s*@ zbrv0=voU9yNA0Q!nS34TY$TeJ&|Rv@=pt=7#gvNvteS?-)|S&v8K_4!6FsCYXPL6m z#7y#Bh^Y% z6&hBpMu+M;sxj4~H&yFUyNZm$Q zo#@}HUFbvAZc`8XO0^dqqND0F^`qsF31@cz)#|7QO+zSAbr>b-s76eqXhqdAw6^v; zZkhrKqNc<@B|X zW=cn^t7f1MosCRW7Rpx5Mla}WL_W7Seq z8Ol{HM~CSWRhTN#j;d9tSI1m!szC>+)}ps{%yp)Ebf9Vj+C|6QXlg!`X+-RNZ19`v-1s@K$qPEqYgTj{6?6y6m*(uswoX!S{}a2($Tr98Kz8huWA-LMc*g0O*yDjH5c8lOO$8IM-QkLpv!d3 zg{C6ZrCN+?wcip`DSA@144tQ1ZmK}9s8*sX9aWX78V#w|pa*nRwWd1sx@tXY)KN8< z8qu(76FOh}Z8o)_an)9Is`lGvYDXWdcAy8f-%e8(no#XV7ih~pre5@eY9BgHTkbav zpdVESQJ1znWEw{QQ5`{->k^Hc#?a5I<0wU6zZ0fO^o!~gxG=qLsokb_X|!tLUh*OxY-1H3yxi znrq5K8>r@^-F3_brb4uXY7yF7$6RbGK|86IqGla)nW-G@tXhH2*D+U`s?e^g)o4W> zRgI|@?WS6X&Qh&6HK0YRjbS+*bCanV9iiHS3U$n_rZ!Zj+K#T#F?X0c(b1}1=sX>B zx2Xq}tM;N5bX0w&esrAb06J53&@_Z9REJTSj(Nm1iWaMmp(Az76oWX z(`c#cOju6GJZqXm7pu;rm336%gIks<5nZC1gw9b-Hl?7;R8!H5s%fTlbh&B<`k!j1 zDGObxnvFKm&+Z&kF1kuJ4;859n+ni1s)gus{nRKj6{CApOV9)QsZnYwL-(ncqfK?L zDomB=e$^^;x~_$4Qw{p7YAqVne(Oy2=uOoIG_L(NnwrpvYBO4-{kE7|(ce|uP`<9C zc2fr$RqaGysCJpU(R->r=qR1%UQ-_$Q|(8OsScP1(FdwSXhwC|G=j!eN71^v7RF5D z=wsChl&3mrnnDw*)95Un$r;luN<1!nubV^r=uFO=61L#^f2xV-RGrBrQ!-jxH3cQ; zOs1OB&^oH=XkTqP!<31(QO!c9Y0KHB9F(J)ii_xp9 zC8koeqiPu{(0ML5RiHf8N_3ZMm8lx-qFRIgrCMvML;0%p=p|j=22&$CNwo=msLR`I zYC$KfwxZp2dD~3w=oHlsw5u*rr>P5_s@jb{SM4$NqSICTP??Uo-!y>EP#r`)szat> zbe8G}`c`$+G=|Pr9Y^cvRySdqMCYnbp`BExO*7~`)mikqZj*DSd31ql!j}9!sG4X> zLW@9b*t+(^`L&$UNouNXX;1Ks}7(e zbh{cf4WR+mVf3i#h-no4Rdoz~qdIPyK!d82C{4GzDbqB1MRf-4s5)z!Lqn?b=r!GP z61MtHiRg9JB=okP%*uve3U&v(cV9S2?C!^r>nd z`dT&LRDeEHEkxxy8%3sK^o42(dR(>CREGYoT8_R`tuR%huT-nhdOBCtrW*9MYAxDX zwa!$JzEy2Nf76+4G&P~`RGZPqsx78g^n+>}I$O2f)PdGHKK!)mM2G8I=rVPqEmV6@ zovwvmQy8>m`hBh=uXu#bhd8K<)#XBw`wKY zT>GsuRii(v)}URq-&#{0>QSvnmuSBYrbhI*Y7^Q@TW&VBpaIoZw7a(4W@<-&Rqa68 zTDQ~Gh2B!_MvJs=kEs{Et=fkgbyWSP0rZaQAUa29W5_g&-c=nzTWY_frZF_BI*xYJ zekV+m=yTO6beZ-$ZJI$-s) zE9sa=Orz)+)iJb|j(OZPfsR$3LJNjhUyHe(SB!5bLcG9d9;G|o3PDqNtoB=KDnqxcmZJ%+TVbk152{w7AGL0^sRliyT8m0_%yp)E^oVK$ z+F!@qXlg=#Qf)>bYrid~R@ATBhJMg~+f5zldDTvIu=d+!>P7>qJ?Il{x!2T(-c;>J z|IwBQOoM1dbqIZ`b%#wO=p)rpG^cgPOylUEsuSo?9o3|13jIrU8XcgsF=LuVpQ_G< z<+R^P!(U`WJVah~_Cxq{lS?FtRIop(j zR#eSJ|IoU5rhJsDT7bUMx`n19w3ccyIzUHNVk$*xs%2;&osDu+1zK0N61}JWR+*~N zmZ~-AEA6+|REM%v>(PGNZ-c23ZKK+R3bo&6Qwz#bZAI^Ezip;=RG`{{rnTQrQy1D( zwHxiP{q~r8QK4!d+FSeWHw~bDR0q+x_B&)6Mu)48pzpNbQPUVYQgs}Cq;)4uljvmC zDfGS8oi@#&Q&eZsK|1C+(>yv=H6e#HsbfwwC85(*lhFs-Z;B}uEm2KF-)g_4%IPtn|jc*s=a7h`|UIJqt{gj&~n=EplJvVs}7^1wBHfaD0)+M z49#fE8T6gjoi)v&uTx@gs-5UC?YGO+jq+7{P_g#g zYwAP0tM;SM_{Z}POoQkk)gknM+V8Mw1eK_cqEhX5%ruS;Rh>WwYQN!fEz?w3PIVf6 z$v@uz$~22kQk_Hp(|+eo3H;4-vT7pwivQ2&Pn(j_xvD8BLARV#QyMx?HM~<)nT|Qb zl!-1-4SyLQqGJw!*)3BJTCAFj!X@DKzovY2m1=ly`>#6Y@UHdCRD`ZkEk;LZza^&d zOrUF3!y`)vYro;WL6)fkU9Vb+{;mC1nX1veTT6y3v<9svc7>dRny)73rw@ zO#|pz)j_nEj%vs>jGj{+K}q~ukT7Z*LvN{$qjX*S6Q)V@w(1l*P5YfT&7gNwXVD4T z@0@8Ky{nqQrMZ)7K|-P_34N)WjIPjr!}I5tDHTntrlEfAH{FzhzERCY*J{66rtnPq z8Py!Lp6-#$HHBx=f3KR4E>P-f^pk37SWeeqnW-GDep!~!$LbQ*m}=2ls&%MTm#E&KM9%e+%>o z<1$S!@1r`2(siz;Ow*`Hbp~ysvoUL$L;I`FqqkMV_q1gSKNpKtlhFRUyve2%bdYK) zx?eSXy4Esiq02jInnL%fPNVl!XH2u` ze$_d2nT~4S6du!kS~c8P7ry5B`D;o>&#H!ZA=-q0y#J&r4LzqCz7Ddq-wac@3@@l= zp?6iYO*!aA)m(Irjw;WTk3LW>K)2|q3Qgf>%(!a!`z9R40^LKhOyPHzk5$W1hW1-- zsz4K}mFP{?DpNK3M70L(tLwMcREH*2>(PCx4W>r)xoT68YO|>YO{un`)%8`@W@<-Y zs&=4-s-31TG_Bf=F4fmjkEs`>oF2|W9}2%AF3?Yh%Q!TnoHM|<~2i1C01G-$b5uK=8U6ZL9U8&lFe$g?vn%dBvs_iI6$J}A+ zM0cxpp_6pX-KHLNk7_SEnt$B>O#SFS)d4iCI%pa~_p1)0lXT1@rcv~u>KJ-d$2@MD zK+mX7qVV&Z_n$OPqkh#Hl&513dtatG^t|dkxYSV zY9U&kfBgMxDn@@$EkR4P>%4X950Z8SBZ-Bp`W_)f*&zou4HpxTDkUHra@GpI)ql%y2GXsRHHhIPS?6)rg3zR>I6EGTMzI5Y??x~s?+FX z{_*#(X%?NYIv1qd)x0Tz@AcQHCZgMQy9%$xUZ!Mpt!fHdPe+w%3eSGHUN!vwyRnWc z!<30`P|ZT&>xWhz3qsurU!R7*^y=r+|d zbe=A6xv2s@rdk=6)8(x)Rih_VYtXv7ytSq}^cU57w2>}RgQ*ccrP_qvQf)T1pr=(^ zQIU?h&D4&bRqa6c>XLAM1Q4N`f(WvSO z`mgpoY8pe6s^e%4eJxCwCei1rQz${V=V{XnnpK@eYiiv&(>(e~HDPD&lhJ+?O-U&6 zjBuSNqg1V%VoF6Ts-~e|wQjm8BP^$yiB{LTS*C2XmTC@KQOBHX%0pYI=A(7BZh@%~ zZKYa-s&t-|Xy*u3hV(LLBsP>{-?YGa= zk4{n@K;gGWK7YY9gicl+Mpx==jF?8zDXL>=4IS0EX#$<9I*AsmPMN0B>8dm6N*(j8 zX%1baI*%^UF(>Tun-WojY7%->$DC|RL6@qgqRX}AG*dddSv3Qlr!8live2!n+2~K& za*in%-KLs{!tYKC67o$2=nmCFl&Z^HWGY5?s+OP&RZC4}=x)_=bcK$&!c>VKSFJ+l z>zJ!eHK$T-ZQxkeewHa-r+G1)&OKZcWY(oXwZ@Z}jU98%P zuF!tFOx@@b)gE-N&Q-6e4_&6(j~3``444Md<*GxdT6Neog055@MbE2_na0sosuL*u z^yKHSX$oDVI*l^;-p0>g(=57Hbq?*OI&VtYmGh^Xh~Ct>N-`y*8&p%!zf@CAY3L@^ zbo8BShA9)>teS-q^iw0-l!I!$@xxv(kUR7;Ecj%VWY-&NTskWk} zy5+Q)+R@)sJJ9cS{dStV&>O1V=v>tvQ!jc;wGaJOwcj*=-c}t%C+HFlnTFBlsw1d8 zH(a7o(-@jk9Y^7Bbl(5jG>N`cokADs5>1s0qdT?V6jLhtgK8SOSo=*kWuO$*Oq8s%k!8w8tE=XqTGd=r9!gcsM=z-smw{@;+OtokO)jAaJ_2>PcO${hR zwGoB;`WGZLnVQkYsx9aV{jSn#YD2rKwxe(l@PdR6QzzO@wF^C_Eq9xG&_Sxb=tfKYs|jiOT3F|?tsg>lmaIzn|4J*lIbGEJk?RAXxUN;9RSb5t|X6FRC)Qx>{f zH5;|+sB%oXs97}+y}CmC_LK5-@m2~beC!; z`a-qK)Qvh+d(eMWdrf`lUe$iIf^N?Pra{!HI)v6x9X5@i2UJJVMyg|`anz+cfwonh zG)Hk5Y9%DDH(mJnu2c9wV!HALm#Q8 zqYPbx8KzA1Pt`1RiE6ef2mMPm7lnHS`1{wCk3Ll`KnrvY7MhCCXR5_$bJY@4Df&XS z4DF>_ZmK~4R;@(u=v-Bqs?k@fHRyBIT2meRTD2biq}pI=MBl15p+sFr&88OgooXvu zUA4{Bj($+>KpCo?rY`iOYB$OR2?;q zp~d;XlvDkMcj+HLs%}+l!Ufb zO-A8fQtp2?rJ{wZX=qd3a?(v1XnWO6bh&DlDI4venuGqPnrq5KJE`WQ6?MxgFcqSm zRg2J;s>P-fw5w_<+DEm_RE~C2tw8VTTveK?&?41p^l#M~Q!UyO}jgcA=-V-)>V6s!{DlZMuH@O#SE_)dBQ3 zUB82-Aylh6jBedAT)!ixQFOlQ7|PUjG;W$eb*hu7QFY2RjV@H3L9eUMn&wcw>O5LO z*HJ>jZ%RZLsV1Qnf)T~;LHdd`LRif)ut5C1@TWzXAkEqt7 zn|1xxnd;G>R2$HcuHQye6M9s&8EvdBx0qVd3#x5s2W`3C)PY`9?L_t3a+j$ay`-juKh-}zM&(PAA{k|`PesG5RK-!U9jswoZqM>QS&s+wWS zL_e!$p^bD@*`^${$^~J2xhPjhm1oLFtEv{DB|54?QxRHCwHTeD{g#+Y(Hg2{=zpr^ zrV6yCY9-3hQB|3$(H5#TXnP%1t*H)erCN_J)KN8<8qwCOO{iM?Z8o)_ZB<**e^uK| z?P#HD2iinO)oJQNd#QG#9d%Sare3tSY9Csvqv|&epnX*b(bL-UkZBklsXBsg*Oo_3 zW9TT=arB0^JYkwd$EZ%BXSC&M(+oOYbr#*BEzg&#`3iQ6d6IPn4P?u^o`iqXa##D=*Rjos}=$PwG4d^-5M)aDF zxyjUwUQlg8H|v;NO>O8!)poS;PT{Mu!_s}7)h zbsDK!vC8=YpTwg=Fldp z^C(M4m9W=uN<^EgCZTh6RLQ0kw7F_3s?t%VnbOe~su?Iey3F&RO<8Cw)ogU?PNC+Q za?#eRdFWZyd{Y71R<#iQtXgC$MhjI-&|12@rKU2pw`w`sLYKF~REhRgtwO77-D*<} zIzqJ;ZK`$aO!cTtwE@-WTs4}S(9x>R=wzLZ7E>!KS8YQ-tG1gu&~d7r=#RR*U8ZhS zq1uDORvkr~>GF=5#?cbh33RqD@1$u8 zEmfUHf7B(KG0mcjRp-$Esm_}c3i;bqH4&YnV@@(9qsvrN(9^1^rZjZ9YC8IlYKAEj zU8$OdQgwN=O*yDtH5YBJ%bRD)M|Y_fpmTJ23r$6+L$w&4qDxd_Dn<9ImZ4u%%S{!i zQ?(LR>6oib)#w4$8uW~6t*H)msn(ln;Ox>s!iw?eFttfwV>x!ThUYc4%}vH zM+2%IXiHt*PE!~9t7KL$l~|UEVp1)X*OZ5LQq4z4YRd(tLR6$$gf7&U zi%lhHf7Md-dtIV3Q#mSDtw3jL%ax`obc|{>xTDRWRfKFF!L~m)` zCQ~yyL$w7xrgd9QZD@&VJ9<;=c9=TRQq?XLo_WvXzos5^jcPBtR$sq;rv4z+0rV&B zchEG1+Es_q>)P*#X%yY1Iu@k;j+-XXGpduQOk18ZO{3RUXV4ODdDb+ChE?a$Sz0$? zpWl>-K2S|UH)`EvQwkbaO+{yG-854=`cgFm-K=#pO<8DKH5=WnOO#{EMM)Qivzv#G z)0Xp11!!&6Le!uw7nzFDI;th;7_D1sDnr?-L|KTXL8ImjxJK2K;b!nJpOB%LJg|Z zXipvUjA<5Kr#gqO*HO)z687b{NYzACq)U`!N=A38rl7sFH1Bl=vm35C~A@cAF67Br>WivFZa)Mjc& zKdE-0>vV}aO1cld` z@c6H345g`#qb998VVXpnsZODXwC=QN24$(vqPMi}oM|5IqMA^|qXk+w(UgSpRg=-) zx&~8Bsc3iAG*qB#Fx`|9q?(DoQq3}Dqdiq~(9ybnb4_`uP&FTgR|N9>XH#KVPPGVy z*Ssu9C^nU#a@A6_kB+&_RE~~Qtw4L|s47iWs6w?Gh3D|{{AW`wI#IO_9iwBeH#MM2 z)kd_bZdXmFX7sRX3(C{&s@2qnx>eiJ#X9B=Qz!bfY8N_7N7ZfWK|QLyXgM8KpQ#@` zt~!8DQynx7pJP3jOq-^SDiJ@p?=kQ z^oGt=!hXLg5k0S(geFvzO(|$VH5G+d*YW-@rgZdI)eQ7M)l5?s8dS|jtLT=KW6DLZ zsOF*dRP#**Xh^jXZJ}CZDn_rXmY^L}OHE~HShXD0>GoV-Htb#zp7rg^lHYQh2hX0DoON;m$%;3fX-KKL|5o* zp~=*Y8dY1+gZf%%HMNE1RNGOa_S<3VM6Ie_XdUgh+th<@QSC(!YTZ6lKYCns04>&0 z4Vs3~tE$83P94>VX%xMtI)*mTH8^gXKz~!6M4RdwoH9+LH&kcPJF2s$IrNt5JSx_8 zlu-Pe64BeLN$5eQ37>GGzT($TEy|M)M|@ZSpj3v<|3`0uwl>^YvpamI7F zRPh|nK|F^u7SG}Q#&ftf;yGM*@f@z{cn-IVcn-Iscn-I@cn-JFcn-Jccn-Jzcn)7H z@f^Og;yHXB#&h_pjpy(+9?#)BLOh4>AMqT%+r)GD-W1Q_J6Sx3?|bnazDve)_?{Zi z;X80Vhws<%9KO59bNE>h&*3LWJcpk%@f?0C#dG)>7SG`)UOb1NkMSISn#ObZ*&EN{ zCv`lBpX>1)ehY}_@H<01hu<*bIs85n&*8V1cn-ho#B=z~D4xUbQSlsp>x$>_J6b%4 z-{|5w{Qeiu;kU(j4!>K*bNEd(p2P30@f?23jpy(?aXg3LpyN6Gz8%luxAS-ozstvS z_{~3_!`}(<9R8|^=kPa3Jcqwn;yL_%6VKtVp?D5|JH>POODmqk-(~R}{)&s|@HbyP zhrbZxIs82t&*87ncn*K7#&h_~HlB}m+Yf&=$36Uw9naw}@^}t^zsGa>mXBW@m9=>=E_Xoyvxc4xg!+nkM z9PXKn=Wst}JcoNd<2l?%8qeV#)p!o~zsB?3Zu^z4hkJtK<-`5P@f_}Dj^}WnbUcT9 zu;V%0-yP54-tl-2_oc^kxaU2d!~OK}9PZVR=kORnJcmaN;yFCN5YOS!hd&*9OZcn*&h#dCOMDW1dQQ1KidwTkEP7*{-p zN5tYeJbo6>;nB8u4v)>nb9f{#p2OpQ@f;o{jOXx}VmyaOAmceaUK!8f(am@ckA=o_ zc;qympK)_|R5tG6G2D0#kNC!OczigX!=uUZ93Fd)=kQ2%Jcq}%<2gJE9?#)1^LP%A zu*Y+FJU*VoqxbO~9_x?i@XUaC?sM}W-5j2y5HBB|(Gbt!`490No-Gm2;kgy@9G;00 z&*6C+@%*#fe|Szv+`}_S;(6X3Pk44p+{1HO;yFC?C7#3cWa2qIt0tbqb8zB0JYy%G z!}ELMIXoLEp2Ks8;yFCiD4xUflHxf$iz%MNbDrWkJVPp;!}F=)IXwF+o^N;C|GV3M zcpg~1e0bJaJcs9)#dCN@T0Dp6uf=nCwp%Foa-d#M0XX(XrcurqDhi3rB z^CoWj@a)34SGzepb20AWd5-a%;kFl^LmBt*jLUcq&(DnK@NCX_4$u9J=kQF?cs||D zi{19Zb5`T!H+0(z&xehBc=l{Ohv(YHb9iQMJZHM?g=hW7Jv>J^p2IVW<2gM4IG)3^ zmE$=)w>h4}Goj--Ja0Ol!?Ud8IXovjp2IV^y@5>+bx8XSK)6 zhv&e@b9lylJcsAk$8(#z{NcI#aSzY*kLNYrypo%LcI$^%D8%cB*E7U(c=ba(hu2EP z^NMZ`ufvFYc-2Nchu3(-b9hBaJcrkh#B+GHNj!(wro?l2B}+Vq*S*AZc$G{%@8ah0 z3Y@ry*XzV{cy&)aht~qd^H1)0K6i6?RZ_fscnwoLhgUqsb9jAJJcn0P#q%X@4zJXT z`%UhAhF5|8f8Cw0$KCOT*Q3SDhgYw~b9k*=Jcn1t#dCNaT|9r`=216?SMs0H*e(T@G8f6d*L;a@f==38P89+;|s6OjC*)3XFP{je#UcnooGBi>9!YM zgBthniq&`yuWya#@M_q24zHb!=kQ9~cn+`2jptdn{G^-Tc5`?=alC$}o5O3B6TyY_P>Uk!#l>r+v|4QuXWpRblVT_bQ7=t zxLZHGi%#6bd+x+@rCa|ecRa_txyo&SUAO%e-S$^^J-o+Jy#G|!!}}-2eNERB-1gUS z{l9Mg@GepD_O^G&AKrl~?&1Ba;xXwLi|6&+{%?2tKi}>DWVikBer)mfPIuc2@AVe<@Xm1Ye3k2q-SrdRqb^>) z$E{!ImTz*~d%`Ur-ia^X-V(Qdc$dGpKjPLu*DZgU+umY#{*G|x?-I9scz48jf8kvf zlmX5INe+Rfp;BjfFbcbbgn@V=Dse4^X_4{m!$xjDR_X1soQ_sw_?@6Z{~ zJG%9EcI&_4E`NBB(Rlsvj->H?z3Y#<9^M-@UjEOnhxc8L`)jU;cXEyUrSAF*?-v{Q z@b0ql{JA@y_qlmrH^1r5PllV*-1!Rc&l~SAyqj-4zw6e|a&vfJ;duE@x4p~U_TF>r zhxbE{*Uxm@yWeduyfbsW{4ehEhj)FB`{{08-pw^`{|#>cJGkUT*y&xBi#z`Us!b5bytDx4ja#z3|Bq@$%u*CE_`JDn&dW?A8yTcoFws-TL9v zG~#}UTffY$pWx0%_!N+M{qUI~@f<#3B%Tj++b?z7|39~W_+*rL{bINNVQ&4O-TISm z{e#^4;nQN`?a#aQ!zaxWMgi`NgII~LF3({|!{Z@2yZ-S)$$PQ=T<@76DL>xWNnizRbnA!DP>PoipAZty|8#v#xBV5}_Rn_f zALQ0gb?dL-)~|8vAL!O!%dHu0<5!>1d^+Y6tC9M9pCljC_8xBTXA`Lo^eT^~{Z*C5smL9JkK3hGW!zZ!F^W$zVck|co z{O{@Jeck!`x4S%VyUX*MJKi1L@uj)>8+Sa9y7>q=sk_HN$X&Fi@N_ip~xov&}){E?eqck@o}e6Hi>Z{7I|pKTD|KFZwuxjUb` zx_NhZem``V=OuT3|K+xSmzxXR{Hohuikp{n`#;U?e_yx%1h@RDZux!O@@Kj8o9E8w zx^51i#Sov*$J~6Bo4;_EC*RGB-0_aP;~jLzd%8Q`P2Ktrxb-i0>le7ovx%Gk=Z^nH zH=pC?mEF9=UA{JVd`G&kw=#Em&UD*b!OiEn?O)@zU+A{Ko7?_bZhI@b`FywiX1D#l z-S&5P+dt23ZxuH;xb5HNwtt-4{z-287r5=ExcPFo{Vuot)7-Eq|(8{(QH5 zqubvDZv85^{&{ZwS~p+jws*hV-YIT-=eqSzb90rO$KCdG+`Nt3U%&f$y3>6W=3ocRZ)L_3PaF z3*Gt~xb;tU>tEp3&vEOo@7Di^yFBl^`CT`^e%{#g|*Uj6xImgZ0xH;R+Te>;R%`dv^`vo^Y=jLbK{Ir{U z-TbJVA9nNoZtigNif&%c&GYW|`5!lb@8+-FJmuz3-Tbke$K3pd`+E47n?H2(yKa8d z&9Az7z|Bv)`EfTt)hPz=BwS@=217l;pSJ|{DPbN-29lEyWG6L`+l~En|E^aR&HM3&8xb3IX8dr zzTbS}=C|CuyZe5#k(+;Yx7QckT;t}I+-%AM{hi~^SA#pgCGPL7<6ZA^`#aFhqi+A3xp_l(Ja@VCd7<0? zt!_Tf9nX+k{}H$RwXUD;`Wo(du5`)-QF-ALaIUh@1cDzCLSQzuNWtT|e6O zV_m=99q&+e}tRgamx?8`DHgh@8+l6 z+~ej4-MpW>{EOV2>*g#suj}SyH~-gNzHi<9Pd5*{c~|%KxW1cza9@ADZa&V<-@32I z!`*zGyZ(OX&exCb_OYrvo=@HWI^4Xcn_qMLPjmCi?s(33=l3voJcqmUcZ}O!rQ6vgWzxW3f&vs|xt{Y=-FxPGbY zZLVM9`mL@vxPFW4C2ropeZBqYZa+`Dxx&rgx!co`Zmw{*?^WFK&AIFEDYyL--2A=U zewmw3aQpj%+uzS_e@U+Y;QEv9_I9)DYq|6Hl{?-?++5=3f4Jk{+Ra)D z!F`EKG;YzveP7}d*TiTR;})aF-Ka59afw@^@A;i`>U7=e?t9^ReO~?l(;vtFRn_^P zI$d2=-PO16B`CiwC_g(W9~9J|3+lfm;Li;BOM~(wg7SWWzn=#7hX?l6!TFgU^xwmR zc+Uy^R|WjEfZsbPpBj|U2+F4f@<#*zLxS^pfd5Aj@AW~tCP=>))VFhx-YKa6q2PF~431}Z;Qy__-^d`nTj2kN!2d&m|6PLP zy*=Ph4a#o|%2x;FCkN#tgYt)h@@s?gdU%Bl_uIhV-vfU4AU!_t_eS9F ziNN1A!T6jWq@NG`Zx^Ju3F^B!sPBxRzRv@GL6BY=`0E?=&rZSk{w(0<2kC~u-)4co zj)A|egX6y-@b{;{-=%@SUj_bl4AMgb{{Eo7ej2pb`hop^LHXGMe_D_p9~{rd!SQqs z{2w0h`v&!O5A2T#?B@pd!-C^~C8&SLAU!Pb|3KjXvcUh9fqgzWAB{o%R|NJ80{f+b z{pErE{J_2;usZ&*g!?Re`@FgY-dx{p6s$HSj+$ zIKRDu_=W`O&Vl`Bf&VRo@~#2DMNr-)DDM}PuOF0e9+YRZ(zSw zVE^x-5x+vWX?ywiS3agR{TCfkCh3oY^qL_3j;6opAog!-`mG8{|4q}6wUP8altz=) zttaVUYP!AhcWe64t)=`fO%Kud?$q=EP2WLj`Y-yc#(R5^zD?6jT7GMgzD3iUY5C2R zMtprW{+qOXZ!N!3)0>w|d^c$NIW50l)04FPI!!;R>1#E;ucoij^j4bwg{Dtbe^*m_ zcZz2{E&o}N{)yTjqV^YQx>3^?YWglspQ-6n^!U!uv^(FY2kFy-^r@O&qyAQF`W%h# zSWRD|<;Q5coz{1#rccuTu)^7E`j8;KT+_$UKL~_>AE;H+@2dZnz>F$B-rNbDDC z`hblkomcz6DZfC|%^LrFO&_D>^EB<+w@%Yrs{LF|-=X%kntof4_drcQsPP}5=}Il1 zqv^$(-e1%AtH1p;{cGj-r8N5Ins#zL`)In2rf0iy9bdCFeYWy5DNX-HW3;_%H2r|4 zt2KR|9$%HFC#nCvHGPwo&(QQ9%1@^>;`@z`=e}BAq4t|;dWatHra}28n%+ap`)K+i zuY)wO?M9LJ8AmgTHb-u82|k={ml?Lf1@CdLTHhs_cJ*DXX?J`-3G6S@wDW(Vrrq&f zplNq}=WBX~*7svgyZX)x_;WS=gr1Lc0{(1GkI?e7G`*?%KU34|Yx)dLyZTPo^u5ZT z7TBMvY3KhqO5=JoR>$YDTJEmz$7p($`a4?Fk7@c)O7|lEw>lnHP#WX!0-fIv(R82A z(jS*Ae}azJR!y%|e+N+-{&rP=^R)a5Ew9t`kF|WRriW{~Hb@^Bqz};aTs_`7ns)8g zQ`6lvzKt|}mbUkXntno$Zv#!c_Ul1u9RF)NUv$^s z|A#d_LDLUu`ch3lsOdrK?*UE!Ld$=pX?MQwr!?AYb8V0NwESOcf19TNr0H8V9clU& zO&_f7b)Kdl)$(&SJw(&zX!=S`pRMWf%AciayC2oGn@`T5G~zu&kMDFXze)X_rs?JC z@6@1twWgiFV=|8LeVVdr$ z-@jlv7`qLaOzfVuJJyh_FHQC^TTh| z{&npij|KL>qBQ)w`|0~N{h|81htead{7LoqOXb~q;%-f!ukqZa>8~~ZJ2kzxmfxXi zw|>4|)5FyMHcjuP>05*HTQu#S$J|V5w1+#N*K7Gi<*(E7-g>^T)$}%+zDCox>3I8v z@-J$beiv%`If?=E?`%!)qWr8NJyX+e ze%f8r%k_R|w5D4%p4}*o^VLz?zbB<}yf10_MwFgFX*b?BR6f%34K%&KraMy_{_5xt z@qVt9oA_XB_nDE+n%D& zC@LcAFp(&_R7TUsAuo$~#t2a$W((1-ENaI!L{V3(A&TBILKKJxgy>ivbsQ!P1#8>e zMu-COCn36&M_q<>zMUytdlbBi=JY6o|JH1R2@( z4C=kwvUm46(c8X*eARzhU7)xV4o1!B8I9kaZsJ~KiT zh#g|0hn&G-9`U&mqCo5-gx9(QJmL!@M1j~%h<4>syJ6*g&MQ6QOCv;q7%#+z<(mE!ms0XBSe9?n2GLQTOlHp?8!Z(GpS#&4f6dF-OgG?6qqa6G6z#^bk&Hi<1D)0 ziEd<7-UTJ)^w*C*Osk0KPH|v)LsUkOG`fxJysXm<6h~2i+429y5LXycsOc7V7Pal} z^;Q@cM*g5!QT{$C)UCZmE(**&Y*`TbBb`btm4QwU)#vV5Rr>QO*r$FhFpk3O+DCI)V9nUq5ieIlSL;A%qGnE zcVGUz(b))5Ahsw)hN#8yU#0`XfRa0aX7?(0mij;)Ol1>%oP@U=n-04RyC6+f8h#!wRqbsJ&c zM1k3nEpyHU*NRb`g)y`{t2TVC;Esh0#?CyDXek%Xq4zQ>Q1ig@1KW%#I;aF{V>qfF z$KJ!CxDA!KO)3CoRYjATqDO@VgK?BYX#WE{Oi`$-M(WDw#3PLm1!6xTyrJdSG0F&0 zAQmSGe`xJygeVY=3Bn(~#~L9D#KDu$G!W)eKBJctu zM1k0S4~{B>xX=huASMdowXT2U7a1W6#9j%)Z{435AqvFaLS!8I#YTt%F*|V_{xtMc zBSeA77bPw=LKKLD5`^EnKQlrUh!cg#Xsery5C!6`1mU;U%|?g<@tzPFZFP$gqCl(> z0ylnD@;Kr|%5~dqMu-COO;O@@BSe8{JGoeIzrzSoAUX+=ap}3!2vH!q$91GEfA2Cv z6o_6$iMx#u1!A)Vp>NbtSrq-!2vH#V$HXwXEc9nv`QJT8hyt-~QQ}@BM1dGml(^3b zQ6P2}!du4s%isHr5CvjaA-sOI73I3^S4M~eF;?q{A3^z-=?9Du1!4~&yd}s0R{`xG zG(r@JsZ4C_jVJ&pi7%o1vJLWk;(xk5WZp!9sbCe*s68=}TGoGz_K6XqK#We*;m^6{BM||vwW2^gk|6wfq{0YMARaFoRU0Ej zfp}7gjB7_nBSe8%lZeVcj*W~E1>%6cinVS}BSe9iFGNP`_A){gi2Aq=y)E;XFB=;n z3dD+-&@t{G$5uv&0`Yc&=tH?~+u8_GApTkOIJPlD6o`)#b@)+@G(r@J)20+_-Cd0k z1>zhbGFo?(5u!j`7}t@~x}%K{1>)yIWLyb$H$oJM8{#@r&PRh0qCgxnt=N$-H9{1K z!-dE=@fjA~nhkrg!F+voG+0%q7FZ*bBquL;)e;M z59PY;Tq8t*IJ#(5=NTai#E%kn_)%SDgeVXj(Sxq+`{}EV5CvirAu`(P=SGMEv4s%c z4Ci0bt}#Lsh#^8`^xbQX5CvlA#BumXev1*JK>RvE^r2k0-D-p=5Wf>5Q;?^r2k0-ED*@5Ff>a&OS#v zbK3vX2vHzD7b4@x?=eCYh_8zh_ZlGzM4P?2?`G6-pAn)!bSz5TZ-gii-GuP&l>C16 zsu7|<>{V54M7?H&C=h!KklgL2)lq;m#92aQ%!!?h5C!7WL{$E9Y-ofi5HBPM|2Q@>LKKKs ziylW$BSeAtTcQp>szFAG0`XK$u@SYs5u!jmFGR*UA8dpu5HAanF_-LMgeVZ76(xol zAqvEQ635{m`7TC?0x@W2u_GU0geVZhgvdDZ9~dDD#0VyumbKP5HMT^pbKx6gyJ>BF3l{mN@$;ou&Wf6V8812DoGO0WyaZ2!-lD+zE<|uPi^T&2#OxZQEwk@=c3AgcdKD zP(>l2Ib3J>5PEAWahuc#N))JhB`JvxlqgV(nHn0cEXU*PsO^7ar#qb5KswzISxu&P zIgHgiq>f;Y0y=~1EY|6yMo^+aomY~Q=s<}A^%JK4yF1-Y)CSV&Ze%r?+T{jT?~uBgRa&Pz zIWg1SL$=cC9`|OtG1losq1|t1%N!}*(CO~yEOfdDS*3Nl4Xx8nw@&x>!cMndyOJ}V zctJ-Hg@7L6I*WBWsS%VYP>+|SBsx%{Kt0LS(CFs>;+gJEY6IzXZ?Kw7?eaRScSya( zYVA7R2V|Sqy>;$%;sqT+6axC3 z>nzskq()GpKz&`3lITE*0@a4zi>ja(W&fKyU2kdw>2$qVO{R9~$?6?a8?#F5bjf?V z{$wki?rd+Sn_!(z6xw|=w#<>@4V`Xl&O)c#mQ`A(>uH^CZ|ijb@H^e}YjaO0UeFOl zA)rBAXR%HvHG&caYFJ51q5~xg)Ci`AMu+_u&va9$4W!fU#cDFO%bu*>AvKlN+I70w zWSi`CldRK;Lc8zHmN`vX-X(^Xrid%v*Lt;Ic^ctJ-Hg@DfCI*WBWsS%VY zP#2b@Bsx%{KwZkz(1^aJ_Fubty^Y#HI^C_TCR4lI!s;DTx3gNiPWLOaO?J9Ht<#A@ zyWh>0Ia0i#(>=sl=yZ>=O6znRTc@kBPWQn&cRKNcjvxvFJ;rqw>vU2hC{dvPP?D19 zK#2nN3{!EZE6aHPJH;AK6xdIs#tdSr5u!kRS(KP&geVZvY@EidsHPhs3Pii2#0(=u zf#_V6*xLwEAbJ!fs*Df?qPGy335Cvj@5Z-4n9bE;qpJ{|B z5Cap0|IBrk5u!j0O%OB;Y@2O_C=kP$;CF%avptxjBz_lY9NQp2kN@emk9iXXW)xe_ zg#`xcL{8FGrOZ2O{}}c)AEMx6GDAM$e>rDABSe9iCWQC&{y55Y+x|v~0#TEwqo*^c z{Tw4if!HrW_{VX85u!lM6#{+g7I}wlBHBc^1C0;`VqxMq4t3_VuQfsxh=!QZFGUXZ zh`C0H0?`~tmGYLyF-C|2aq9j&#=WuOe=2mW5u!kxEkwqr5XTuI3d99W6o0AvX0{>Y z=oI`NxP!$z=#6#Ff;siTo36NTbMmPnKeQbh~Em4(VnLmAqvDFh49`~9N^Wl6@9QE zQHcWaY=Y?R5nCG}3dD;-WW1}e+M*H#; zh7qDbd{LA*(+E)@z9~waWrQdYZRhZ)%V^!RjSvN*Q&Hj^BSeAdUX(c32vHz<6(!Cy zLKKM2iV{CILKKMpMTzr`5CvjeAv%>s@+LNY>56jQc7YM1Kn!7`_#3dhvJJfSb*^ZC zp?MPpZ#%Q)Tv%bQ8O_NeSG|kOhbZ_M%aH%hEPwh3pS0azgeVY86NEo6-)Mv=5G_Jv z%p*4$AqvDwA-t=1cgl6!Ek=j}@uQ-|twx9facbf?{3Czd2vH!eKcLtZ=?No5fw)zO zj3fV@5u!lcov6c)>M0{cf%qUn_~+wkBSeAtv}jb%7$FM8SBX0Os9rZh6o@vp#oFo( zBSeAdC`3kEy=jCf5Zw}W_)&dmgeVXT6NKMZYm5*DqM>M19~mJEM027JKdL=Pxoe;( z5I4*#)>eBNAqvE8LS(en6eC1|_$3n?Z;tm01{J(lupiAGh3^$S#kK@Rf5HEBn_*r> z!RznXGUrS%3q8wOxZ`=ARfjGpa~)Tgb7L%sJBz8{#4>NIV4b` zK>d@cUJuB#9lD|0xIEf;*hb61)Zl-*?QM-O3T*j&?l#^e=`ZZ6j1UE)gAm>=H5u7f zZGP}j~( z6<<}170+btuPSDlH&O7mD_iDB@rFj5z*%UtJy^x7ivHYQYq6>jLmaIr)U>C>RPYT? ze^oKtq7wzCnwhLs#Xd%e0$!{7lJbT&gF3A4czB07iB|S6 zR^k|~EDF)y!~umfRDUXQn*;$$6sS8=DgP2DVS*9`>fTgpJ1TLT91*aU zoF)DENr%yXuAK={AfJ)?KvW4qUw5*t&InN;UJwG$*7OO}@64WF1T;bvh*yge^NkP% z;%{+OI_CYSuX!Uxfp|Yba9`2L78)T6#3u>DA32MR5C!7P#BuoZ%3>o#fru9HjX=hX zT5p6X5bc;Keue76HgNA>p_Z68QSjE8Epsr%hH6o`RBc)fL&+0**h2vH!03XyRJ8jTPI zVz?0K-)G5Ox4t)MNm2vH#B3gO-8`XhU}MI{Qv z;soK3>_dzY1)?!Q_#=CT5u!k}3gOLZ{>VPm2vHyo6C$Hutu#Uuh*eA!AK9zfhKvhF z_F?8t6uh0lmN}SWLw`G+voNyHVik|yz>!9X0&$ZN86*2BBSe9?BPLQt_9`Ppfw)(Q zjFEk`5u!jm6xWe5vX3!B6o|)SB4uPBYlJ8ef6%DXM|RsG?mk@VPT2vH#3Nf7?X{*e))K)f$R#>hU=2vHzD5h9~son(Y45MMG; zd}O!Fqm8lo*X)zcn<#k0N2(Q34hkDa_IjL!k=>b9JhHc4+mS7XZODZv)YO%oWsmGr zEILtOdNY$XvR4}+3PfKaGOpRD8X*eA03kBYz-dN^0x?jCjFEl15u!j0jfs?zeTEUD zKnxcmV`QIcgeVZB;yO}B_E|=V0x>QoQbzXKMu-A2SqN{`^`cz2onwS35YzNH(&vIr zZDflAQIjD2xnMIRM1j~Z5tV z%mtSkAqvDzF_AJCTxNtQ5O)ZXF&A8JgeVaA#&x931y>j$3dBP(kun!tX@n>ck7-ot zBYT*QQ&AxPkRbe#y`vGLKs+OaH{bXp`znh{6o{7+gg>&cHbNAL*As+4vVU%bC=l-m zkukD=VT33U?+cO9udXpd6o^llC_b{Ig=ts$Yt5S|c>9vQ6DcG6W+Oy_7%D`@$iBr0Q6PrLb)<~!Ta6F}VpL3|jO^Qt z5CvkKMwLFYC)+p`1!8i7@JDu~5u!j$6Cz_|-(gXS0x>H=_#^vHBSe9ilOX(&eU}lU zK+F>&V`SfLgeVY;g~;evzcfM=h(;!gkL<(PhKviY+4qXmJ2vHzj5+Y+{|JDdmAYKdJB;;vY#|U6o|etkutLXWP~UX1BA#J*-sfE3dF#;j+Bx8v=O2} z42_ADk^PJjqCgDSsM1GvtBq4pAVwt!e`GH=LKKK`LS&5W=PW8wAoffU{>c8b5u!lM zND%(We%=UCAZ7`XF|z++geVYmgvjVuFBl;T#5^X7kL*UaA>)FP{i1mj1#gSlG6z#^ z7}*DN7Djd}t9WD&THBE=h8WqRP}2&DDRX4MWYLKNvx=Flk^NUAM1eR#h>UCY%SMO- zv08|XGw_NLqClJ_M8?Q|)d*1_&X0+dk^PzxqCi|CM8?Q|-3U=2u8iwQ8QE_bAqvDb zF_AK|-!wuLh?_L3^pSnCjZ;w|?nn^+$UemgQ6TOWB4cF#&7u+o;*kX5kLqr^d9~&VG z#DJJc8QGs0AqvDmAu`@)yVS<1C=f#vgg>$`GeQ)I;X-7L?0;EQqCkvJ5dO&i%m`5+ zCL{=dWPfgiC=h!JkukErFhUfF8A4?At1pcZ1!5Kx#Ygr$wjtw!k^Pl<69sQ`*fIxG zY#7;j&ceuE%qkw)!`60Wiy=m~s1BqKVrSXU#{X^6i2~EgOxDQ$j}f9k9417@HT!EL zM1fc(M8+BT#t2a$P7oqvWJkMUZo#pO0l?I#;GU}*CYskWZ!9oC=fRZkukE{SyZAx+?62wk=@=1 zQ6TP55dO&SV1y_Tj|h=5vez?06o|)#$mmxcjSvOmNhXSq?3dVvj0;9~C-Wu>-kxL2 z989rcWWT~$7}>A0ibwX4wH?`Fh~6!#1F5&zS@y{8Y|)7V^FA|KBYS-#M1lB3h>UA? z7b8T0_)>_BGtkutQ6QpvzAwla+1-o~1)^O{q>SwDMu-B@S%{30-NOh`AbP}gq>Sth zj1UE)cTA*=>KnzF_{>c8l5u!j06e43}_q3=)ff$w`{E^+u z2vHzLBnW?G_clTlh|xl1jO>k#5Cvj_5E=cdj}f9k?8!v&kv)rT$hcr+Z(`m=!P^YB z%)t~JM)tm(g^@jnRXnnXuI7Mu-BjDkf4!_7+Bn0&#*686$g3BSe8%9oLaEviln$ z3dC74kutIe7$FM8`5IOF$bQ|%sVESaBnW?GzhQ(Z5LXJ}jck9M{=lLV1>&nEwgS_4 z)BSNe(g;x?$_`>8sX!9lt-kxX69GBDoFblrMS(pXiWHqM?O8L=Ex z&9^5t-@eN8rG1sBGk<{xZ^u8%Y7%+)!n+QS z(7)ha5z#*Z75#-3{tbfE+>c|mHCDof%6{)RR%_cRB)f`g0v8p4r2P^ux zCi)r2Ox-uSfz>incd%Mc>ONM7k$QyHQKX(=bsVXuSe-=bFRV@@^$M%ANxjADd{WQt zjN`bN)Jv=`C-oYut4aNh)pexaV|6pB4_Vzo>Qh$tkouC<1EjuT^(d(}+`5mGx`fpq zNnOF}8B#xIh3_0+&+27Tx3GGH)Say6(&e{&K$`*W26PzEaX{w*T?TZ^^~i0Q>zV7F z>yz6w*EiQMw`FcXZtGkwH!wFSH#j#mH!QbPZg_4)Ze(s$Zgg%;Zd`6cZc=V?ZqMA5 z+_c<`Tve_nH!HVKZok}|TvY4}Skj4b3m^)^>}?)NA+-&axJ^17C{dvPS(1{@2}%^G zk4sY0kwJ+9^+icaIz1>+puQXy+03{04!6hlV$bb?BYDGy(E=Zt6fjXij zB^NJHqCg#6l9CG@C{dtJDoM#j5tJxUrC zg+<&KtR5isH7hLQD!6sAh`X58A4y%#>KRg3v%(_oI#yW3-OLJ$xI0+QjiM#xv)asR zH><;}jN2a_fjth~@W7r2_FmFw$)-#CF6p;q%OwMrY`rA6WZ;rPO9n3)x@6dr zot6w=GGfWdC8L&%UNUCMxFr*oOjjy4wNWRmzJcYbAl2D>Z+2IbYxJXKwVdol1>jw6sTKD zQZhh5i2`+3NlFG2C{dv9FGiv?GOl+V;f%>E*C6gd1 zQJ}soNy&r>N))Jw=Eda01ix>{oglw3eTi2^mbBqbMHP@+IhD@n;NS)S)FQxtRea3e=G$DY@|h zB?{DWB`LW{0woI6$t5Ydp#mie)EOlyx%mPm3eXxD2TByE%S%#n z0|-hKsGl?Szp;38K1(mjMjwo;1~z|5cAPEAL?PM{Y?)(Q`X82LV>t^;vI(sIi~ zcR;)0bG15N-e5`gIWI!Glj_K~`#niL%nD1gQMM$z$d+U;+mfu4Ey)JilI$T~bYV&M zXI5B}{hbw-WM8tHOsWG%fF)T^R#l`1u)>mTC@U<hzM7OhBMSfjYM&C6g5>QJ^j=Ny)?pN))Ke zN>VZjf)WMl>XMX9n4m;~y1pbOlPf4upl&Tm$wUlF6sWsPQZi|S5(Vm4B`KNUL5TwO zXh}*g2B1WNdZHvH7YevCB^Mb`qCow*BqbLlP@+KnwIn4MFHoXDy-|{q3mqs? zpx!M>$wd*AC{Q1iq~rn$N))J1OHy*N1tkj9S0yRAFoO~Ws;s%x0|mK=gAxU*eMw3# z_@G3ATE8SEHwU0Zf!d%XB{veFM1k74BqcW;phSV%yd)(zD4;}v+NvZaH#49_f!eMl zB{x2xM1k6&BqcXVphSV%r6eUcRG>tG+N~rdH(#Jcff`?uk{dNpqCi!aq~xX!lqgWs zOHy(J2uc*FnM}Fm)Bnc4_-l^uN&OTFZpqztye-E>A=0%D(&BC z{J{=x-7=Tg09dzt#&_x+>G4;2u1&7pq{PPv{ar1z7YArHQHbReuCCZUt5gO`6sRvt zQW6*_QJ|t0Ua<1N3F?1FS&cH?PO$b81+FV|{4Y8*$3L=;lK4mHHxb{IcdvTF?nmZL z6qt?JGUu$&`h7VIt-l4UwALSCtzWw%4g;9V*Iu|(b`fJeD2PcIa0i#Ehlpp+Hx;eX>GZS zwdH|3y0-juUD{F{;P^x#mT6pFv9^@TK#2lXQ<9RvK#2miA5+qnS^r?bN!DJXz|}KX zd>&aYzR8-_%C?#|Tj%-WI|;>Yg5Pt`VX@{4+sx^N90|5C!66A@FsWTjU?9n2jUU?Z-xl0`W!S zI1Y8@v_IbnQ6RpFiIlG=U0{SL5N%sAsr&ydt$)5RG(r@JPD~U(-@Vv|jH83`Kiw`e zZ=&F>J6q0$1zMvIC*Aqhvv({?b^D3=5CtEbG2|bEfA%gmLKKMp3Bo^nmlz=m#I{0s zXK#jg_I_%FC=f%0K)==FSWdZayVM9#Aa;(4l(To45u!lsnm7*s>|JhzC=g?rD1P>) zvJE2j&)ya0O%%NC!IpDjiq@FHNq6>Aj^Rr4AqqaK87k{o7Inl^aoRSO9smDlMu-Bj zuMn8ks)U*55my-@3Pi2eL1Rz9`sB}?FBl;T#Ct1vTzFqz^S_Grq7kA%tPvt(u6@Y} zQ6N4O!uzK~@EG51FB>5WL|d8wXtZV!uNWZ;L?TQH55FaN9KdOz55C!6kqEYoRLKKK^gz&D)T`AXX{frO=qTgYB1~Z8MMu-Bjvk=}m z_m6y%5u!lsN*#~Jphx&ezK0Q_K#Ub4qnAuJLKKKSgz%2TpV#*^LKKL_OcbBjTiJ$; z3;x0Tz08{^cxz;7M8U^l3~@W@%<}_hPW!1whyt-n2=A!< z>-aPyM1eRVLHO74=|+eGv08{u6;Y>$+AnK2m$GQ9_paqLj1mRvETO#F$G?{EZG$KTGWu?v5u!l+ zB~gbT)lwrwftY?ou@Tj1geVX*g~*7i$p}#(_D|H|N43%jQ6Mf(5Pn-7W`rmZR}_uv zhen73@ry(qepE*rAqvDR3Br%+7$Zc1c&lhs#~L9D#CuG%=9lXI7-iA6nEraR`k(#J z8)EDLzP_NE%A$QpeaWh9hc;2~vJIl{%eu{l(KjA+yhSPsk$x@5=G{B`5uRX#C=hLq zL|@x5?cn^W1C_7K=Ecbv!dRcGl7s8Qs{7E2s8?B!sGHZUcrqFP+mpRHI5_y+bNHjt z2pNwLMBy-cuw@RW*kHX0XW>CbUsmyhicRR^Q4!JOr0#9Xw@ALb>OXcCQ`992b#1}! zqPG5m`b5fi+lixHe4@Y%WX7NT{oBryj1UE4Xi?&1BSe81UX(b+2vHzL6(v?1AqvE} zqQt32hypRWC~=w*qCiY5N}O(lC=fM8i8G841!BLV#F<8j0x`EJah4IHKr9pj7qDT{ z$G4?ix1DW-C=d-riF1q)1)`Y==W}iLr_k3d%J^3<8d+hKi^3Q^k@NbII*HZzk#m;q zG-pA6SyStr@l6Zz^uOl(km}av`o={qL&l02yf~_FT#!Gs->7~|Yp3KJ7t!zJ<@z<} zTk_3^krH4wv8y-DfKP1!u<}mO}9(7 z?r_dS>qH6PNP#xZCTkec2a!<+HhUAA|ehJfCA8&SbUL(T|%nX$bwz zs%x&Vo7W(RAboNjkKtby!F#MQD*nL==Pf*jwT^FczF}FunFdPpBJ|azwUg=_>l&z= zE=X&J%)CB>T(9cL3ek09l@{H^L-W(>mgXazR2B>C8d~z1&eIC&tipa);TD+{rg_N9 zYcbCY%yT}|^WeclT9+;xQj>3K9m3PW*v17@>KhN5Qa`V`u6gBgNq!xL{sX z%6MtMc~RaOXT_J6f3n4Vq{V#H+Q+OD84g0^U*Dz}^Zc}VekRki z-#5oksT$Hsoq;>;x>Wh8Rr#(}`S;8!*Tr*tdfilR3O+Z_UwkW`w>Hn6Sz+p0pH&jY^|&y&paX!E@Lx8k|RJXf0MJ--#tb>?|L^SuAJ;`s#g+-RPgz7@}x znCFwt^C_90{UJ1A%9J53H2pN>7c?{^CV!N+u4tm7meevTSW(})c-j2L#ftNX=GS?I zUTsHonH|yPnMbrPz5Oope3N;;`CIY)Tl0L6dA|2s@%$I_{Iq#~=3DXHhaL;beASi} zW|elV(q@%q4R!PLi<=r2dyRIdZ?NI}E8g_+JbEAET@)zpx=czqP)paq#?Q z%TgQR2|Gd=oa5?R@<3CK*ERDQbN+K%_Wi<^eb=z+M^695b~#ela^6r<*RdKw>UvfS zmM>kpa^dnuJ{(%=AxM5=F}Sj2d;{GR&!}r%oa={s<$l{nNX%@lZ)h1gva)5;@`i>Y zr95-_<#=u)i@MBemXZJLYEnzRYP6h=?f?1qlAV&cN595-nA-o}_2{$%f5&2bixp0B ztvqk2oxTj+xUPZT-@wDMH2i-(?<0#ZZoALECvEq&#r9RA?W*zzFRyQ2$A=={R%><` z;_fBlMq8irzWWZhp~cpn6?eF?&CPWyr!SO#))IHT!u)TmdDh9#<`!?hz$B&GEVr@nYfL`S4ofWL!ng~c1U<>B9xwmjBiTUEL( zztbLhszn;N;#uF5R=mJsJFl=6ai^Y`5|Q~GbVGUx$HI)8@aNx?CcM#NyDqy4ztawQ zw?!DY-u>T`)_c@qdnl##vZh3I#P4QG{G-Jix8*b6leT=(Vtc-HTmFxFB2 z@-0?rk1W?^s;Ho+5%MtfU*`F2r@TW;UXN3~R2>h`Lg&!_BsK5YjazFd>)A|vx3=J^ft{HA#hUuMg2lA8UQ zdH%pWe`ucnZ(@$!DgE^u zOJAL$zsJVWS2!k+%^vhgV5Qi8xBq{IKX$+;ldrMrMrtd2+WUJ}{V4APR`~FuE#DPT zUyC+nMTZ&O1JvTb30#hTq?qU#QscBr2PbejI6jOUD0%)dLmRgLluj zpn7(&`0;@me~grfwZHj*D&>RU77L&I*Yj2nR9w$)bZomuoxT1w7mdQ##gQYco0?nsk<&I5 z_%QmPyg^WK6w-e;UFF&xD)x`BwGE;3|KMgMc&p~uTw0qNN1>vr^lIIreDmm$BU@@K z8(UiI8t3zqRJ*wFy-0WA}Ay6W4DN{@=40p<@5**e1?@$u{}G>Vt~w8$idiWn$i# zK5q$qy6y*!^Z2RxfQo(4MI_ob;bSJf#Xm_u)us=ty@?pUwp``~NY)w7}Bz(9RYYQDAmK;SOb2U`#2iNn#2|vlD zweJ!BzOpuliv9JYvc3s_`^^sd?N+&XpkjVU;&&katDQ*j7SCb)68DVeriJumAe_Yz z(~Z_vP_a)pFBV=G$MYiwdhNKW5pOroqgO-xAjuzmerP_SVxRq}EP0%j3w;;hcd~YZ ziuv^OCo$!leJ>Kn#})@v><72m{SxgxzG>OYq>q1@52)BjU-Gdj`6#@uCi!!Whl=_1 z`D1+b;=1O7Nfqt!h53MreWcIJV;k!0lJmx`<^wABkv>K%8|ODHUyz?s*G!X9!9gLO zUF`fo#lGl9G}=DVuGIN3DefW92&m^I{Iyx-`ZZMSFXKGToG`Y)1^2ms+wnogKIltT zQ7#da|COesuW!s3RP2l1&yEI?uhICKZ{Ff@CNEwU-X@@}E-_zFv9GPDtbZad{hCE# zNPK)Ft<9lgKgoV1k>K9NcY|22uvnmCAL;Fkuh94&WK7*_KA>VB$v6-JzB*Fafe^=T zb}Ue_53Dt|qBe=|f#p~xH8qEa!)f7x;lu z-H5HN`GSglxqe+Z*JIrnuL^y4@HO6iLB+mYUyA2N_x_;oLE@QYKA>VB?%EUg1LvdY z?HV8BRPzZH`*iI!-qv~U9&0vz$&tP@Ru~@psy1Ixu`hQn<6}&FFRjoa+Uy|n2NnC1 zHICo!)BKe__9pWW75h)WccPc}3q#~%?qEKkVjsni+1*YTI>a#_YW|>Nf5ng4`73?Q zhns(>*#B^*o5xIj1VWwCRvM z&C~^s;HUj6caH!S`&qlTbn%sL%d;(BsJI?CuNB5iZ8^UnRBp@7%m-BL!#(2(`lLOZ z3tnt3aEvy%!2CkRekV~`#;3T+4IhI#hvY z_`S+vg^K;A_n*XV4ac_rD%WmMv5&Pp&n~u-=lL3o6)LW0cRG%Y=U3dHGA61-Tij&+ zp<@5j$p1P#qjt~7N*>Q`Rv%PcU;4VSCVwbDJ1hvB+rOK&KUD0;-9x1GH@Q(Qbj7}& zv3>>>`%1r#=wr>`na>6m4^-@9_h_BYlj|1+QKHY@XYB?h{yWmo{^E0nvgo5V9i#GY zQ8)TG1Sz$3)Mg|4Jf@tAysf65zUv(g-hJ#WR%6DHm-22zMV;G}(GS1SkIvB6Ct9yt z1?`bvL4Of>X@fm&R9}xW+K-{XgdNq}PVJIqv5y{6yB_f09{y?Hf%Z5K)aQ<)BWc70 zKd1|3o#+pniv?vHP#R@!^UKCiW?EU_vWgzvH>!x*5+^P4(VDIsm6JF8$Yfn}E43xc zJ5!oI9h}%ux2%QU!B5GXS{;21+fLE?VmqyA1?((6shQs3r=R3l>Fm0Qo#j_H9x|z+ zsjjtDIn}gg{f){Z9B)KlqTx6ZJjL2Y^IK}oZG^lrWa;HuRYl6Envd3?@2nq{mv^OD z-PS={%I`Y^qqh9JYpEZ_&Ovbezh?Rnc1@S4UC(ahuRHy9BBwatI2N4mVX1v}Dy@zV zoh)BYM6F(0`|XD03w;E>pk3eTNqP{aTwm=)Y1bcNw-NnC}g9MD_kFs6;@hj zA7Wg;MBhbv6ZTiwFAK^#6e%MgexDuc`}E4XL=_wLr!@ZPx=dqT#vy)Qj*qRK7fXk- zXbZ$d+X<2M^-U=)M@ zb)X;lbqq>ls1I&W`Pke!vi|3tqeePMn12S6EgzZodbIm?lqcs&GkLx#iw0AE6D^19 zsC{>%cEH?#JF50?$DBFXfIk=$aWua zjJd(7;}`c#dodV^4Yb&$~m`P+~?Ld92pdDy#@XOrXP)^su_2|Z8II-w| zyU{*AYv?~cqS0FSk#Pa#_lb6=d}^YoYI!3{T4dhn6OE;OAavd2pV)zqag@=U^5~Nn z*wP%)Bbq>Y{gBtx(#o0C7tn_%Qho<~NtEvT;wq&t??GjLpT}H-IBIKYUaD)T)mI^E z@$|5EenV5s^5(q6)uU`O*-w)%Fqgt;&(}+V~>q7HC`Y!q#{#?I7 z`E>fDsWzH{J#DUkrQ4N7Rc2S-4g6~EUvhn^p>)!BSv1p_GG^R3olW`vINc|M-#6xY z?zQU}#Ty?(S+u{}C1Z%k)-b)F+?#AMe^={%mhSh{{T$sNK>LJ0Z@goi;W*rwMr?>_ zF8yKi$MjH2XO*G7v&xpyn6%^ce7bw%`NZN_7SKK|4>4k(`E8AR102_)f_lrM#gxVd zW*P0lz;21!aetvaXXomxr*gz_5ba@)bBUPU=Gp;%k?;IAs-5%e`U-r6Wv(BjmAQV| zN6&8)`9(a-bbm1I!66Uzr_~ph`Slf+#p5N-r;BAvjip6nXx064-5)~xBp>$~H*V3V zhSFcswme!%KCmUr3eOS7|6yA14^!)n$B&x>mZ{(HJRE6t%G%6tW3c5bS+sqvo z`h5S?I?AJCw2pz=hDTExzK_*?TAjXMGUYZ2V>n*@j!vIT@gC?Xokd6Ul)C!>el#*> zmd&j+yUM-DkI0uB_lOPY6XBoM3vPVi-Ut27&3Pw-p_GgD6iVaIoxj*ed9<4Hv0=<0 ze>HLL+?Hx+8W1)}JCCdRoXS1JO&QG1#)uux zUz+b*)y~C#3#AeBZM1jwp|7}ebw`GuJJk>Bz;U2WT_12`Dy__o7gzt?#G(FQ(jN9h zNoSp(!u}GUA8;2CA9Ha(bjN{m*tzo~{l6^w73E_aP3y#RS`)TeMTu1trq@oJUc)~d zTT8zqJH2XO{_$9v$HeBFZ@xKd{MZ>|$5+o|lyu>leGrS*D{*i?T&(Ao{r~C)yP4K0lA^ zwh?`nLwS|0^w5-IKpOJ`McC5Hner^kxeN)44|QKhWz~85 zfM!|UeEP}{+h`tt{Q)*OPWW)|NDS7Bx0B6S`aGU$ z=6b=*p!l7C(vzC$OgbNTQTfDXdJj$OL`$~v*>2cmi z`P7C5$EkUCoWf3Ax{TYQoR6CpVk;!3^5}l@F}49~$5ve@mPZdz-h_Oe%jI*9ez$i0 z9nZ$fxzFG^jO!D~!J8VcxW^e94KM#S zmAS23Fp`Vi;}n0srazrBNAHVh=waObhU$8Z{-#ic?j8sC8p|k+aTwl@Jx*oVP&SnI zxL1<3Q(3fckurCW?dILzkuB!J-_t%V5B^&N|ISBRnXB9RdQ$!W$@8CdV=qPW)xrziRffrg(+Yh~YKbyV$`x-o+>Nl}B$Vk5~%J z$|AR3ak1<}wyxb^U%0-mi2mmJf?eE3ahdbw{9vEv>)j&0T$%H=jE>LwOY`-QBEDSN zFkLUbO}1!%f9n2y+NagEHf@l`xnn|E*mizd*yb+&G|rW|WBX8JU!(hv zbpNsLKhgcCv`>ri`+RJlY3!ft{tMlIsr#>V|8Ls=f9=@5*4V$%ebin0XqoQIbzeby z9NSRpzwy|M$FY0H^E>r}IUjw!EpcwX_RH4YKCYYG(iT6vaAOsH8-2Jxl_FL*=dd|N z(}`li=H@PEpR_Hfx5mU4vE%d3&XmT!3+-W#I?xWX#m`{zuK(f`NOGunX)RI^_Ysn3$A@}XQJ#x$ zyWPOHjgF%{e|>R+rRgp6rS5%+is%W;=YQISkM|FrzunkhdGtpwZwt+Pit_NSlsfvD z2yOUuoUTNlN8U4(H}T-*^bbYIF+FSPN^W?&8UJf*AW*EPzzrBn2Rrtz)@%3q>%P18a7Mtr}W`%$OpuaqZm zY3{wesg-`~OONMu&gUO~=J;{`YMQ1rt)L|?d`WsH-n8cQn<#ItUn+hpqPH~dzbE7J z(BaU}KOh_W_dIw_iuAuUT}kmG{U1$Fp*IO7u69uy8dqSDCwBA?QBfZGY?)2JN$%or z@8vnW4wT1hEwSYDwq8LV&Q+(VBjru4Yn_kx25_vn#&DXNr=Yx2`np$c{h=%Qz&DVj zJvv3*C{3#->d#<0MLi@vZTwQKg$vT`19=;eO=ZjM#r1SvXqj&3O!B$y=hI4gO1>2j z9uTWz_&jBmiMgJC@ilGv(s}ai0)2Tys@FYNKzbueV*!un8klDb*SHz;GsOdY%!7sY zg%hW=PkDs#?ytFBdRh~=BCmsS&Etn2$b+RZ}GrLK}Uo&^1W7?GB zYN_RUQ5LflFY;$o9GdTDT=apm&B#7nW8*W~qiK!Zm&)8mYi#w0Pk~j_rv_LbZ%%f& zMr};SxF(>k&hk8z*HG?x{g!0s@^L*!e)_u;^t}}x(7cj&3HyvoyKgZO!{Zk6c*kxbs<7igz&#Tar%6gwLAVH zl<&tn)TJ77g-wMZj`2b8T1T&7o;N6ylQ?G9VK|&%Y$C#sl7G-59c?`;8bTJIo_;pN)TGN(1royV3ko5QdiFT=b(d4np&A4sw(? zwP^vrc?sXh8%lW`1leIu8AoxrdjfrqSvH>X(&m{S^uEaiDs$TiYh}zc+Q2lfMrXcj zf`1dP@VANhF*>1h6nF2POr&}y(I4KEdtdLRskIvqp?`S`_agY*pe(ASe?qRceqp^_ zH;`8zx$j^_Y*P`r?}kgdO|(~jCB5q6e6)??>kjyK(R6)NP3+r8RZYtq7mRDBFP7(9 zC9gv?etC1l%7VQ0qH4PHYR_oHK z*Tk}poka{{nMr&1nHkn!Nx$^3sj7eXoM2yNeIMY^?s1)k@t|$9%5eS8Dnnamm7(3! z$}m^sIjCG8%lSDQw0j;fmwe){j{Xog+9CE+5zV7~Z198jc60s$(y()z-`>s#-y?X} zJ@!x618`Cnzfb(7ezsOcWv+v}`M8?$f_ab6Rb@*&$6!9k`J=poc_z+}=NjCj&Su+q zeu?ekIR@9K{n##^XJChOUfEJO*TngS^G%#zIOoLm#q&;_7tcL$UOYF(c`_#Inke(yyS#=G|>EgyS8?GVI0bYJ3eT?RO5$h^ty9328zMj%tgC zeV_D#raQm$C7*Ke+f%aM$x9x-%taq)%1tZYNUq8ztjQujoa|ZxBjH5P` zN5`0_-|vZ#hwC+eR;KNNJdCN{VD+vOc^FH$M$17WP~7X6MY!K;snw8p_LqE&3(J>K z`Fg#O@^LRCS#k|uM0w%$8u#i0>3Y4G%G{QVM}c0j*$dyRV{K4RwwUwroji=qEl78! zR=~Ywf68_sq%~z0Q|g*~F@<=~eW5YVB#%bSOuES_!}W^+Ij2kt0>m zs&RQkt6c93vc?qHQ+tkE*K2=)gZQ0=!zh*?(jP6E`TS$8i}UQ(5Qz5(Do6Xl*3J#x zbsb4*Y-kQzuTS99uK7z^e}j=G}3HGd2TGG zmAP@o&+lmrokASO(rVgsJk+num$!bqs=ch=4kf#S_NQ@#=Q!wd1tUSPA4gHXe%ppG zUi{C?<|Afs9=vnK*QR4BkJetZ>4kT?JEw1ktDoa3pRQS1`S|e?n)54cjYf4YqBrOH zYUT2=Mq@+1j#NabAM?H+gR{l@YyuP8}*Q2biDzcl_#NU6lylTp$Uqc&@e?GU0e$?9cTjQ0@ zrtga3{U6s}GpURrzMY8kXSw`Y`Q~{|^vaO7CC2JLu8bc_3q#!@n&a{jd^3GVjDEmq zO8$_1gIC{yWP|s8R(L{U8MQk zqv>R{G&S0JTC8O{psVTMWut#9I*z@*u$+DrNIo}k$F`)feERa%Wy|qhSyz6Lmh*t| z+N44Ad2G|E@!Nc5VvF4SDp9IL=e3d{X_P%a?Hkn3m!>Xf?_(2Td3^wVq}@khK%?^Oi4) z`BTXTx99bZ%kyIw(&qtWOFzBA_2W3spz;ah_$HT+0b}G+%A?g0=G=U9&EjSnlGf&6 zel9*2)-Mtc>xC1-d72++$$CNO@Tv5}%6duQZy4G$JuT;d!aZR?eHQ-J2EUt@UmzcZ zDPI=-i0an)5x)yc*ORj7#5j+-I7TztMAim(QT*Noh4XRTj}_$e0T}Znm8Heze7;KM z=?8%GFZF&%Hi-j}<10If;ytqP0?VI6ol5z^nz-yV$_v*PxbLr|&m2ytGTX`u9b$n| zkVE&WxGy?`7~B`psiphYwD0}Ts-|Q2e%+0_HJl~0dEJ8hl(Wd6dxj6~)&=K~_SXgH zQrg`Q!R~DO!@A-;>}gAu#rI$CUgpPSi+iT?X`it7?w=B66qS2dJFP6fAC&ja{pV{I zEBUxsW1W`TcJiXIH5Cm;8aToCNP^Naw9&kQcKj*%cK4w;Mx-xAebh#aal&XDnObk) zv+#sT6ueF;dcMUDKg*TIBcFVvf4)OcBk&$H`iV$BrgB<~h<#N4Gi^5?qQ z^VcwV4w6+ilx(xg`e&4-_eZaB@v(H!-jv1djbr#Fj)CHKYh1*R`Pm&i%J0E(l$S^M zVow{MKOo7xpI zU)(#qIG2tPaXv!zxHumrjo1*+uSv)8#C2XpV+Pm2`=~r*wKp z734ka*=@I-q8+W@(mL0jKlCrOImQU= ze(&2+o}2UC`7j?;wm+QH=H7w(1F;yFf26(Zt9XYC;{rU&V!k~36WL)y3|Lnp2KRg( z9LC|(WCM06*~M%{^o+;i9xRQ0E`$BEvSD3Asm`>z{*qz8L5BT{8TPFi_J7T=@19}* zN``&64ExtI?E7cfzmZ|TJj4F24ErrI?BC9?@0Vf!ZifAs4Ey)gexAPP{Vb)?_nz1N z3%Y+v_b==IRo%a?`!{v}H{HLZ`@ie{AG8nq&{@y3!w- z)%8*IeOxn9mOKaaOpNleh4$`z7M`<;Xalv!^%?zPkk<$3CudUXD2sY3liugh<)$?? z%Hqi-_5`EBW_eg!Ho=xZvZHW8O-lSch-;A_>ynQL1 zY}2x6bIQktd!TaK_o6iR{b-+-2Or@*mKz&sW$qqpR_dBo*3x~}_k(pmO!qs|KHz+uloi$)m$^E}Yn>Cg&hl=q zKY^V@d2yXFi!#?Xax8I~^Sg)oolN_*Jovz|!hhHvu7CRdYYN$=F;giYn``UUd`{)L z*z^o$hB0D?^OxqkO6^?yds7-QSJU3rC*x53%*^mJOZ}h@9EX3*ZhWPcxv}Hw--lS# zzpvIalyuhlDeN!t`9VD+GzRI{aX)l@4&{D(IJ*NWAKU-!?=@UbpP#$mYnV$uvuDrn zYgpgvZ#QJm#&l1BYkM8V_Ap(P{NH7;bLV{){gJ*qwo}n#k!Kh?la2mHL+a%M&qIcj zP565aI45|pfcq7EzKwZ-9)cG9Uc;`G=f0Z{KS;}WSIVO?q}{U^_j?WFD33nF=Eo%N z_ZlWrp8LIqO49Co74U~KJ^%mkb|v6-71jT~m#(xxpr9;@g#scXmPLdL*fdFLAWdSD zE4l`SOqxpu+iR2A+V56 z&l~S+pkDUu$XN(y?`!A*EPG!A?J<2erL;a^ZKs;hW;i#dHet`6?96=)eUOt~*Woyb z_cippI2aGdZ{;=FIX8N1N_ow5^{xh)`{Q>GWU`&*n0L1H)%0bwKV1j7^=(S$t|3SJ zZr841z9R6MdOTX^?~Qyr;@}?5yYV*Dda^fgPI3?6e70newEOo02Ycw6z;tWlU@sPh zeSLVJhtmxS>x=gS)-opR3sP@9%zrmc>yn2xtR5T3&T{N4XFTthc--QID@-5H=K2=TXIb8din71s?1HEk3&z12( z>G%SU@y!E{XZJ!PT^UuAgER2XTHgddRE2cznJ^DABu(Yxa+#~9UpNExQ8S^j6JOy# zJnU2I0hf3NusgxkU;2Y{0XN1J;D-UuyF{6O%AmhF-^f-t#|YdfgtjZ#kG2CRQg$>T zCuc2mSqQjhMB_TFj(Z!f3;>sD7c1ugw*7lb=Tz2%j!(87(RD1&(LW!1PzQx>1Kvxr zwJ$SAj|%C0R`%6Hpl6%G`&e1WnRmC~r;pZuwkL$E|HS`x%s2`eI`$#%MF=NY7aQXn z5B*cZ)eo8gob8d0oi>*6pDX-%*|u%+`$@+^%RO+La^OXc&KW8N)kcW0L z-C}n6{UBU7U_S->=UTqo;v;)`!hR5`AH;WJC&|EIzf$;pER?~zA~M|WH>Kd77UEey zWOEN}-NR_>xW9MAU%)*)+EiOJfc-I80*d?hIzS^|r0c@%Qhn>(F1#4;*}%K~F8x9! zdxRM02yi%OUfZyGaI;|!@ox>h%lJ7jA!QQ~-a*;bXz(`y@6#2&F&6Ue zF5p~4jQh_9u$w4SMEUoD#$|fzu-`f3+7zEZ4+7WPyLotG-0ywVeEVU5H*dl@H*RFX zrW+1WqufV8vvT$Dreltq?{@!SAG{VxNPkknJ|Auk0uq-*Z5$qrHc<~;!^DPx{`QTv z(a`~Hr;W1b0@s1Pd~0xOzMDtIaNtUHo9|?f3=VJR-YGZp7vfu(!f~IFDQDo^_BK^W zw8b9H3BL;~#(#E}Hg4SUUwJdSj$~;y7fe2H&C)avZ5+TwV~p3g<-RHh zcT*WnOi0L&yK=O-gaq3AK&vNKG_M~d|M0N86x$#Aj(PM$;4n5H9O!dfz+6En-y?Zk zXKmwv+YqHRNmci^V$0sdCN1tz&h@~1jBhwx1SU`C0^fnNc)e@!B}v&g>)HXpV}0t{ zykYe)?)le>9qnNu>4y48hX?u_M%L3Z;vP84>4cbj(5AE0{ivXD;R&563fv>v-l8!0mYE(h~k{!e1m?#}s*79P1p!HDQc5UIjUP zU&c4|xm@0_#=H3YCCsFRg}^^D{@Vrf-^=`ia_4yHO7PD{sxuo@Kp1P!YWpks4P{x!bvNJq2X>D25*#J(*uy#J89&Y)w=eBLiaI?TVY?>ycB7HITS61=Y|rYn zyXqMeE$ekIpO7h0<{n8`Ynzd|XVQtgUza!c1S+rJoMdGqaxWDdb=oWGYVPjFqM)RP zu)W!8P_30X7{oI_Cf}cwbOW3(t0yDBk$%L`+}70G*4WaV&eMI8wq;A5pq}p2JDR&# z_J?CWb6=bXnhUFV1MG!{xnP0ojbNuMj^)3FVy!!U-rctmxkZtrQrAw1TNBccB^ zsT*$&`n!V--rC&Xw9dDQ&STJ-Z8_Uv?cdl&+y3dT2x~t>y4&$%|MoV%@u*+UI#Wb< zhtR42`*wucy5EU+@!C17qN|jl6LKN+MAY@2;Pbjx+<)y`-UYbT`)-6)AM)IVAN9V6 zZ#+e~12N|0qq@EuwAA%Ic<1Od^8-#Vl(OzQ;CvB(^TAKvdm)1~{=xjB{68<)4C^I| zy$9zYTiVzuG<3HtYSn(Ml3>qBQ;$EiDz;EHFdXf~op#=4>XP=;c;LCGt+~0W3l+7a zp_8JuADsd`|1W~==sC*|LZ6WH3l{J?7JKM5hMg&CiBLw#x7;QllZB|z=(;)S4&6hdsdfkxuIJ zY0-saPX5=r@j3jrb>s60t38wMv-t5FSYO~9PaYS?m0CBx2wLXDm+;Qfo4TR-Z}-Q} z17C<2`TrG(mwHpC>ispqnKqU2b%e>w__8t{5*ayNK(Av<<`MHk$Dk_XTf!UZ{Ccy- z#X3xVJzXXFj?in{HzanYbS^)k_xA!_)#+=?;0M4_*DTKu1CQ6?Z1@gEbUzMsxElca zS=m1=k^M8nGdIKYixQq+8XoSkwDJ79gy%Pghx;ro&+kfj9yUBka*F4X5}rp54~ihg z^M?|iKN=q9z18i{B|Lu-o+7)`z1?TX9^CJPwhnTCB=4Wd`{(lhmAro|@88S&F?s(< z-hY+%-|#Lff8DQ6n;t}l*>wD~gy&y?^C8bc8J@>Wc;v&O;Hfw})4ExOG1xfk5H8Yl z3XR6aS*f3kxAITR@J}z{pAq=0^>a|jHqKp2cxDBjio54ss z6c6v0<2MLD1D!b!^&sBbQF3gv2QvH1fbrQ95QOiad=+r6A@a=mqKP$YFsbg~-vFHA z-Fb6Ckp7#1bKPt4-~?`r^D@2-*j&K6YyLbu>AnltVSshjYVtkQ{S(s72MlX7oD6r% zzPsaI&c6fB_^B(y2OtmhoIBT*pN4S5Ag<4KM2yqGJRxC~13{Oae`C6LWXd2I5ul|_ zA1Uu=oiTcWn$WY|15YpoBVE#X=#+ZnF4a!!0cbCoR`Bl&`afiwWI+2i%wG@c!OqwsX%zINd%?Pfm0+ zwk@0P-{$l*x-Tk;ru?qcYo9ctop6P>r z(%D=!SppoNVm~rm-@+9iwz(D{eqpu#RLIbFlmE5dY{h@uZZ1Vw>lEou!;f<#ZG7V? z!s%Y>JkGXb{=UU_$YJ_A!04~dY+{1#qxd$-pn8B zsAaVU|EXWnt-BLD0fL0rN&RK|MSYb)S1exPOxMQ}?DSjQ4Cd>IQ7kQxBdm zc%|q}t{Jf|LhgU22lp9X44JHFr$Z*&##i9KwvCtIzpW3KBFy|!o2MS6-Uc`ymI3)K z&*j3S_U|e2tz6-ut?vThWeD?qh0t^UGT4I|JKiSuoOgoG=Bytt)+JqlZ%HBiIByx? zXj8*DP{JXb*?ukujr~^3284Btq-P99yqwe-gK@xgT*fwi0^!y2T}XyEBCLCs&&B_m zoH5vh|JgGIn*q}^1RNKik1+Q3xPIWJ_^qn{6a!u zPdsnIathhxZP)EDc@60?uCDQuk+q>d4uH77mgPuW4G7^uzE!RSE;|eXOg_dpFOhov4g!#S(Z|ZR!-Wo6Ea2}~xR$X!f@O)U8*hj~4Y5HD_EKd6`^07~f ze6bxb9?!zca5LH3F*g!m8)X5m@1{)dcfDD3qkcO6VY@)MwhOo7zmA!d=gq+l>XvzO}yo=X$!X4F2vj$vx zIp;FN?XmIeI`4oC-)`YM#C;4u=J)-4|)^FZQK}7#gLoe#D?aK+MU;L?_#6RHDP)$CQNkhcNa?x?& zXF`yU&N13v=P77rFU{rR)`Oh9u#`bmm(Qo6#wP(piQM+ z=2gA~xb{_g-w|QxTX3QKC^wZlpG9D9zJh(Qgnf64zKXCf8(Dgv(pLcE`86_<#)tix zH1=0aJNql%Hm+g%J*3w#{T=PMSa$NKWo$%JX+=4=?d*Cy1gvtlgO7m4 zZ%;Tw(zW@oL5Hv7$M1iz9x(rHyYmf%wQQ_Ce-m&%#diz(vuuc-lPEHTeN$;h_w|w}DUlhaVO4 zRFWSDo{Hm9yZ2dx)1_tbF#a>2evcm?D~tKg^i!WlQ*`OFqHLx6gV4D$1+SK= zwlj|bPyPRhx0Z{R{Z1?TNhq04&BCd5P1! z@o9-TmHSVT`!BqUV3d(AtC$~ZpVq!ASPLuSsRGVN%W<14Q{UcBDdCwKcsw1;uSoWE zp;P^*AxxcT;H~lbdg$fs0(g;}St5rxrh|E^HlX#Z2&Z*Ni#((vhDUbElp3Q$8;hOaI4#$9u53pG} z=SK@H`1?$!`qh+QM;>!FpLlyVU&HonKG(nP*?iLI*?jq*I-5Tm^!98%b;_R2-yJZ` zEA{E?0V_J2zX#wPKk3vE(;={sPR|?9=2I_wHh)jZ$ezvL3$X0jeA?g6oXw}raJ;29 z!7&znZ;rX*{FaB!222xSzYB@o6zu$pnpY0lBW>7$ z(D@+zo&o*!o6yfrr?s`AYe_O>(s}R>r<1JPPotyCq>t_EA)?#i&`rOqeWcT^skyPW zp|iPXc>^~F`;!*QqvshNpT0dEi$s*FT1cGInED`@)>Q*mw*RV*Xp(IecgI z^&)miTOrNq`b^RF2&3ysPFLh1kL?)w47yt$g`ESM=ie^+{6x=R3V{W=gbLV{|%UdpZTSQjH=-fj$dFpQB8< zy6>0uEMC^;$`VT9U|2iObY<)MiQCgHFQQb*0@9SWw84`^*Q1TD&v1E{=5B0l@8a1C zTi2hzbGkYaJE*LuO=;`PdZFlhOx^}@h_5;1sV(nC+tafvu$HQ4NK#rc4;w|NV~tMU z{ydlFovjzmJEqg*d1l0)oYKnY)nd`>IYzHDUD^7}NquIe>+6#3>gDb=b*$x?WoQcY zY7xDTGkTq6_3{~(YDZ4nLA|z*zO_p~;*ozlG|J^p#YWV)7#DAcx; zIHzx?=*u&rI&Sv*qSkgc;4CE$VDd=i>B*y`rmgPYo~~K;Qay8oIUSdYj?aURI{(Re zXk7Nfr9iH3@9!>07@TpJfYc5)mS65@$k&%n{Y| zzxpd@;lG{VdLhC(=R&$w_$8S4?cp0w9vA0y-24#M=gtN#=ZAXn&e0dllZJS}qdpSn zwVck-RM&#J_SKJk<-3-VP2IqqtPwr4JXcP_v#x}PjeU)z#m8C7Si zvr(dKJ7}vcPi6D6h)77;ygAcAau)AC&Pbk>T~MCYOi| z;@D0_`LTXbKH;pd**1!DH4kWIq`QnVt5cH8`Nl(j*6(eZv@yOb0JnMMdFztb2(Qfx z(rY@>an&VPmC%nP{v!FWE1_pQP(=Uw68iZi@~Xy0;4*#|_NKw;{~;9eD4S z)a}i7U>?F4t3K*<#ooY>ezO*GGwwU(ocK8SFi#%tC6zFa9bgX>+E&s`cpBH)x_dRb zJxXvJJr38*`8^8f`Y=C-?)DnI*oPqxA5z0vgw^=)A;ar$tK(j_HD%cOYyf;@f$E=3dhoyrD>MUC_L zO;754A@o^1TC1HW=W7^VHnhe=gy)rjVZErYcO0o5)7wND|E%Ov;7~R`*A{IK>D=bw z{?qP|vpaU5zX>qYognWk$<2WA8}Qo4_%u|KTLDk6qg0YN1BQ)m%XSn8Gvai=!sjKQ&?mdcB$SsKSN&S{n0&ZIe;r4B1o*hm zL}R2Q?#CV<1wOmig!ymxny?MX?lt)YV7jk>V-SXQugPcdU;BOCYw|h3FvdvjHTfc7 zy4U0@_;2@`u>RV;CSOHZ_nJ7I*=zDO;Lxu(+C3No3wtn_PP^CS8{{#2O^CO9O*Cxx zntTs5cCQI(bgzm0Pwh4NA@Fvu33bZuHTel(npe8l3QQ` z6Y6F6n*1E$>|T>!0+!utLOa|stLB+z+#dz{|HfXE-$G7ye8M?7?lt+Hi-YlSOjcBD z-9*`BwanGAhardKmTt(&`uSG?Uc*n@pEL2_KU0!Ny^r_L2s7Tl;=LcvVo$|?`skFyALZEjmzeIU z$=`wFW9L~kUh+oTYVywvE&aqY+J6hJ=FPtlrjC!}&HPWxPW#R5#~AM_{P@_h(OH1$ z7>Z>>x;prJmQMS$D&Wjd&idLoJ$*V(rX!{`#!0>j$SA_GpJqBZmaua?lr>Z0Ov|lH zb^)A^(rX@;(a#R_zN}{<%zWH6BR`u5wmi>=KKb-l6Z*(}m|yh&_duBWHBDsGhKll+ zvPegqFQaH1#JwEwh>Q8GV*ujwGB7VE`Y>O~uVdSN0JrjbfaBw2U~VGdOdq-ha})Fz zSA!P4e_WT5zOfH*`n3iDKN4rU0dE=@!<6%0N^{uc1`hi4`KKse3`B<`#s zF0^GIgMoslsf2LD&}QTdKEu5n z&OdOTA(ud;U*q9?Ll*Bd$V3CvYGH)yUB8{0bO=Um8vi&;OOb^F>B>MxW z&%#Vil$nQE1u<0K={pTjhq(q4g~a`kPeI&I#Sc^Np}pl<9=2;t8{3chz;Vsjooz$w z_cYO$-;>F&6(XW#WBAYn?w5C0`AZ5c%&p(y&O&gD|K|Sd+{Qe>t?zj-!rB&+=OFyJ zes~Dqc($`<>86C{qO~erfO8aa&GPAxtNkb2aE1?)ICai|?eFo(G_QZP{&4c3kJCQo znF#ZzdPf=6q$0^H`+(Ix!H2!CFm*pKTx&@({G zbU#bp^W}Z4yq|-2PENr*P)IXm+3|_XH<#vW{Rxo8d}ChN|4f&b!SnE+vQEU0kCnxI zXZoqn$tgPPS9{q?w@~O@nSxi#RL4Zm2cGF~z+3xcEz5a;M_q6mJ!tsky zG|py;bFqsv+7o$~02bqnyu@kV__Rcv%54$3r{Y}%ql}b|oBUAwwD#2oIYm6}fb-FE z+~&&E&&#}^gr_s`csiC}k?d}vQ~kRTrq0Xo*7&3yabvU*Oyo~=$iHGfDS^1>xC6*84Ss*%iyN-6KHb=b852ZUBa6bRdd7#yh89d5#i}^pE z3UXT@VhfDEm-9e<&?h?&6w1qtVIP)xpySH*cjKMs0w3pr(h1bLqT@l6oeN?a*tsCK z1KGKtlK``GK@96$&?5ZTv4GA6H35dPQ)({g6u@*Y=rsJdb3v@bb}pzDVVw(dI5QWt z6u9hMkifzm3Darkf;z}!=7NZ~b3q!mb3w~NW9NcMqjN#>KQ$M0I`DQbh&pBGf>r^h zd8KnfX8~3;7xY5Fk%x_T&P!k+ot`(&1yL_M7u17rb}pzFu5{U|bw8>R5qsQ0_2(d~~e9I2m^t zj%|3vS%2J*Q+4bxCNkU@6zS73s7n+4?2snYM*SRnv*_P#0B?45OkZpWaQfLcU$spd z0S+f&3NwK~s_~BG2;B#wO$0TEIhk z>RJ}h1tmP=lkmJ;c${5=F8XijI>9;?ah2pKWmf_qIO@*u8z|rGqeULoAA+*byt|e>2&t~uAl`lQ!@mg&Ah-Z#nnR(Zb}@0^@WS*G$0vb1ktnwf9R ztL%TKMf3e__|H6hJAQmDKl7StrVe+e=?*N`U--Wl@zn0%T zz@sjeK$M==+d@S&IKa}NHB>Uq+r~2QIFm?U}-Ws2@ zVa^ZzREeBViyX$mbTCiV2DDBU;k3SJ{GS0I-mL%?NyO@Rd;jOnz#=10k6d`;r5uc=|{Ykm?m*4HGB`kL}T zT7-;up(dc3xH>RO@W1Udfw=3QZMUkei7lUulXQgSznX3 zw==#bZRWq>Ykm!Kvg3Jn;w)llL_z8@F&w{}6ch$M~D=58KGg5Z3;bts%j2ra0>r_G z_iI-1{w`?w{=U3_fVb7B7xY6JeSF&Lk{?MtN>5tq&UEDCu1kI*^y*uXPI>6#AO@5Klhw{tm>ylpy{Q=lJ0XH}3LcaMDo)1J`uW89|GJ26m^ND>* zEaz(SJE84#tE}03j^QK+*I~f|`4W6TFOQ>VvCIM%T3Wan@N;Q@zlVI^NBjSFeDEl6 z)So($?h)#O@%3YT(P zk^|9}@u6*S&%-p(qR+%Q2>T!Kzhw*?`3JZ@UQK2q9FlBv19g=6U4Y+GNZ{ApX8}Kk z(*$GVy|gg$RAvK~wGEcTw4_?y6*ztH4c7mt9RAaDhrTf5di!qBWq16TT`U)ur!w!s z|Jt_fiT}23nS-#lAEes@KekPK@r@^si{+~IdvDOP?4N{pjy_Y?j9=}AXXO~@zWDLU z+FZoN`4P&eJqBHoGMziobgNB26>=?IDowDnfHQ5-M__0CUJq^1f3K&SJQ@E({qggZ z$v%K_Zq=6qZKW>R4=~q_jQQ~X2)B$i)z%CQxo-%Q<|%-Ah^JwA8EzjZ4cjj658*i# zj=hYm$^oFuwp&jFENh!=Q#e;T7dUBM-ptZKcepB;s9&oXL&&-D*4+GBR z;Tf|3>q^K$7KNTweUz&JXWm7GpJ#q8;0nXQdxPtLUEQrquFl|O2bjzEZvlrkL9)xs z`nHE*>f3eW?mQ3siys;}UY~06V^8~niD7)&Sp>7a5O~wTD7K>w=)LpR!blX_)hAx?)+@wyVi+A9${~jzdzz1nA6t*YwM69VCo* zQgZyk-0^$Hs;&*;`i8axdKch)z~Y{;svEl;3mEeU9YrW?pGS3-gHg}2_4^ROvh|yD z;q%}-9SWR2ZY@MBPRU$2>-p2c!?6dG)&@EGJyxv)L+fj!yg5HwKx1#;I)BGTDHbD& za|-3fKCzUyV60L~>t?O}*yR~Wi?&hE#Q(!heq{TRA{_e)!nH4A|Hbf;lv6Nv3Ajvu zMm=Vmu}h?jW0$wd*yRY&>e%I3_;1H9&qjEMa!cyee9)V*Olg+!JPOY&9tHaB*yd=! zvi8MtLn0O~FQt`o#2DMf(zsyN$g!*XYa_$ss~TZ@xHEis zaBS7$MH2&q{ql03ctoGEs#Itk3mpro&%tk3B!K1Y$_Qg29h)%xT*h2b&H-o2S;rg! zml<;qPFtiO$NxIkIx)iCyzNN{Q`YnFwz5t}m{0MYAIpY@NBe7|wf?2OBPTZa?&*lt z$RVvcmSEv>p;)-Erw!H8`KAjYUwu>l-{y67IxRw2^M$wu{5bw=O7sqKjU)%&* z)}Lm)bM%>fBwdIH|Fy0%?+DktYr%i5n;h>EcPf5-l!tN6L%0aXawlBdh^6??uuC6e z(e#myaY^~PJZVRmva>uLfb-FGw;{~*ya4YtuKml-E7+n*m)(K|r`0z5qsgo{8o*ID z%abim-}ZGOzS+JE>$wiEdA1C2n~voOGacP{+qADhm=DuVS;FJS)hg=@f$N+PWu1v1 z<(-aqlvhnwA`Cc?{%11}Oq!=`WjRDW*sot;(!BaV zO!Ibi4{fz70m->ej~}kJ>Kn9m99^xL%@VRQnJ0rKIiX>_~P*?=_*D0XcV#_yeh;FTA325wk3VpcK0Ml=0G2DtU z`gJ#F&;2-^2-~q7$C<2WTFGV1m#hSA@j$%W`z(Yp`%vS!m^`$z?bHQF*Eb0r`+v$} zUY5!7W#-4d>pBcW2ogkN>4Z2BA3tG+WAvoSX%qiDP1K zKhFoumNR{K+9dNY(xTh;@l-Ydm(4Sd!>7YvA98%;WuBEQ!z?SsqQmDne;Bgt_=o#z zd>db_e=5fRwvQMEE%Wq6c-wtAW1^4dcfgta{`E9}2hGPJA44V8|JxlrYwCw`{Pun&k&fhWa)<{Wz$mI0?vTd~A zq&=*Vw(uTncc$-|+ojh_r@j0m3+cnXl=1H9lYN_T)a3N(Ta=M+OvAC0^VoZgT^gQS zNN+C|_PQ)6q>I=6vYli6Cl%7A@65%GlWp2oE!+Wmxx3WKZ!DxYx6n#HES`kU%4;d6 zGZ(e`^tVn%Cjy!N_CmUXn@dH{&K!N|-J!0$mKDq*chgB;l_Z8svXDI1;;~arv-F!CYr762MoK)^^n)PZ|hvOPM?_i1%v3K>~eCPag2XJxv z%gf{V;9|f_`${R-anRX59DP@10erKJw>I3Onws@axBPQTBd$Hn}1-?hdX{#B5}{C=&xUx#;&&*Y<)3HLBBh3(pVF@t`P_y(gN^F8A7 z-{iJ9O&9Zsd7(DR`5@9=FLJeftgapxzo$)HHtwy^nUBR~Woq3#8+C`jJZ&|3Q$hMP ze$9W>F*ol-pVsQ>$`Cd0RwIKnMd>S^&-XhXt5Z1gS-5vN*&v){T zNB29?9%&;J^2_r~N!}$qd0f;@>&0E5W!~H^?|bmh@tJzD2tKO6Pr0of-#dIiJ&ucBI4UTYb%u18FvG+lrd-0o#JYhbtUWM|&e_OUJLzb=9qpzcFJxJ;i^OSra z#193Te8{~E@)j1MWlVcv*;>2!DEO!cWk-3H;?~cr`&#~HIAXw|&;A&J1x^iUt9{|7B7yj4!@>%@1`SLl0HGa~42ESyN zO7eNW@#JvTm@nD~e*v^Cr!V52qc=9cTx=d+A#QGH>S=FlJzea2MQ2MlzC%I)#;COQ zDM@$p(hgr2wkB=uJ)OFM|AeF<`6w$mS)o;(QH76YVT{S~2e^-OQBhc~5`Z4?#{B(W` zzT>e+*~ZL)PxE7;{R!UnHs|To#5{Lx4Cfbr3Od^VE|9M_FdH_YZQIX*qdl-~U>oTA zV&H#)|9r^9^wMs$?f(^Ub_^BUPV7-Je1xmL{TBbNeC-EZ+>jq|X3p|PoUNd}-HLvO zbHrR5ryn!c(^*?H$bCM;LoGvVhJ88CO5Tn*P>yLNzCpmfMDGU9c7e<)wY&EKwfJLzgUe`2oP1hTV#{vy~5%hVusW)<$b%*vG_0sqiiFj{v`PU@b0k^0xIcgvH*P zddGX6jQc&z&D_xTKkmacI}8psjHs@k@bD$Qux>YZ>%+^2`m%AmB@S1QKIv&yEvYaQ-&8?>kk*BZE<>E;K*UC~uHUgN%-sJN$Pd&|3- zoZ`Yy*i!&2Q(o6I=7J`dwJv#HHtqAUA{vwzWv~pn$0sJ#!)g1x+bJZ}=S$^xfM@IS z!+>S$C&$wVpicfCIDHo8t&BA@Yh9s!J_25j!MI*R|DE+f#{rMwKkKU2yFVbzpVrqn z#>ahVfb(H}C43wBw2uF|kf)OTCGb=nkH-5qz*%pV|L-OI{}BGXKC!RVvzu=PE!&ts z$@{PJ{-?bEg?CPl8JFa6kfr@B)69HhUSyG|t%)=dLc! zYQ5SJd3Oga#u<5u)4cI%i8z&8FLL+5y9h=ZOlz=7lOJl6)~5D`oFbkl0nSJBdYda# z?{n`{!n1GS@pLS|BH8;3o$9|I!qoXGcx!wz9(O+e0VQ&tCUO`D(_z!Bb*c!b^+n@9 z5WI~4Ac=?Ly0Y>~+e<7T##0d;)Rxo^)#iw|`Jr@A2b|9nJ-c}=#`b!b^fMr{WG3T( zdxkUL!iF~XaOjjh%Nfedj5i*Zvz-4%y(~$C>vJCmUe0u8V-06J6U1rvB5(}Ma-7|5Il>=K^ofc2cM8 z+0J=@X3QSXPU>aPb{>jw_H5^2fMw5i z(iYR}-=(*zyRi#xhGQ(X2^|yB|8UBgvD9SWFFyiuvf~%dL-K6rvs@gEhdxD_<#QOL z_Xrh?XHeVdZyX6(9M23u7TX+-9WKM~di?lkpR4T=aj$l`*nj46kx$1$96Q?iq8LXt zIY!c>^N6J7oD%8nS;!dol;kS zAD|0YjU~mFeKy-R5PbVDnLiNFQB~Anz!^miCUWC$Pcjgfj@PO*$I*Mjm$? zjA{Xwl5uR58{$6>Ff4S}a4d$j0sdT1>+?g-SuhPJSU5QhZh^%$4{aPku5Ey)D@*0b zgIURmN;8N|+vvHuZoHrPi@52xw#oTs73VL~caws4Ysehy&|gdYkXT~^KmE0(*zVc9 zsn$2)rc^qIhsT|Qj+g!V643Zep`Yi^XsraSaRS!?Z!k)mHFMImMhN(EZZJ)UGnMJU$zi9eYU1+TGp2g zW1I%?u-$D1Bik^pF`kd#R{Z#AyZcaB(`a0V&(bbs`Z$+cNtSjkZ}fAy)ugetLDnnZTzfjmw%E zdfJvl8fDE$UeMjsiGajEGg-a}As5dsNqgs_7O*!gZ|>>n>S`7EtfY|>%N@!y+w0lW z*y`6=c9m~RHX$$U!uf&)8d}`DVPJ4G=i^r#n}#jaAU8W8S9f}y#chzlk%+i}ALgK& zq0eIco{4f`*)UvGHpH<%D8tbPi*VX6anC{VdU`cE4RP{02sV~qt2lLdwOgdX8);{M zvlR5YHo^ZoH_(p%){pBzSo=)Uwc(dwE$;<<<0-<~^;-HrmTy&9%k^jBJ0XWOUGnb6 zJ15i36Hs^Rs6IdYe#&2gA0Mp;l)?HQaoE>_{N+R3>ChwM(tIqVGF&P>$hCOhwc_Hd zLw%r*XHiGs8P^N(f5+rSJ#-AgHpjP@1^I%U>y>o7JOgc6Jd{y}%ce=!xOH5@bf~Yn z2JyMLM4qz;;MW>GNK+P9x=dnvDUfwu)wX`;IwXm~! zX?u5bTwkp7_{R379j(pX%}tR&`y&@m;BRejY-pAB%BfjiMM{4&&Eq>88o6@sj zx!UdOG+P03;Dx?@-syV?Rt%5gsAvbq(yJgL^h@U>{w?^eK#8dx5msbJTLI@o9BGQm zpSXFTiMAf=@1nwf$E~4sj1I5CAam4j1?31Q4|d}mwl^TJeMQP zhdj(b?Hky>2Rh@={Tt&ro?p2HwE67vzR0D(Z7D!F5yPKC#p$6{OgnCEU%I9w#J-hmSAO|Kt zI_?%&y__qc2iJUHtMq#+uR%CFR;E48##s4E;Pg4HC?_Vdr&G)J6wnzdI@79qX;OKL z>V$N1?&B)x@>={xp^%on?HjKKPVI$dtbOC_@t+TIq#-UlUP1kIV{2_VJHj>8W#E-ENtWaP76y=Z z+S5ve;m-o%?SZh<0Sopd{_z=}j-zbn)1K+W-mP+$=;fv0UBeTjeKk6Ij%O8k+`J?X zyE_)qL3kQ@2;Rbbd>jp{af`v?!QQnU!^k3RAfS$C1Lu9}rimKA7{jT9PV5_BF%b48 ztOiZ{xkJHk?(_0OxIc!&d|?f+>^y+*1^1Bguk&=N=@h~Tyv!W@9Kdl;36_oNYWT%w zEd!5ppB-{(TkmBo#fo(AT1+!g?jT^zqoc!m_Gg2JJ4S0fXwW;3do^HVq#XhdX-M%i zzuJeoHjjB^?#P3G)}m#Ws~AJc^Ren^OS#VVsKou7kdm^Ya46+Yys|%RvDAf zupE#VoDM$sjIr^46u3M4!G1H>r%Qgk%*UNaOWH2*GE4BYk_$b)dw9He&{+UY1Yffr z9^Jf*rLTGL1|LtV1&e1e>XK``oI*V3IN0{HKkyaZ&#!2Iz$oLKG<+X4@azv|UvC(pSM zcOC)br#Fc#?O!eKZ8Cq)KACWi$4GZ8cr4w^Wv(nQBaUMo4*g`{GyUXer}{be*SPq$ zzh7w!;zK);+pokuYcDIpE%)E2dnIsoqRH3o>B*}APtUVYPc8v$ONy9n`}E{e(vCH7 zniT!QtYjO)(1o9P@6xuS5%~jHI26IQoP1pa8%8jVY-)?!)#v&Q;=zNe+&Zv!9oE!% zAq{Luc|ESP7N>@NzeGOt!;7L=U#FE1ILO2Y-9~Wm%}ry=7?=928=e^KM|XiSf+1kD z8{)?KkDe_p@|KPCNO8dsd`O!GoJG8d5b}ly*%1uLKrF57WT{hB> zOD7EU1Hi|Nrp|9GENGC>=0C~bSVa^3=@yP6?(|L){`wYtGV~fYd zDRAjSP8E1=R$lU7_zWPQS+-|v zdyHbPy##I8C{3Yd_%m?x(U#q)bj~20X|V61IpPez&&QiU6Z0{>#=(5Pi!x-g z+1rgQM2OCF5zl=&8d`tU<72>=)n!fh{h&!#=9niR_qcRrmiV~_`2g^t@env|VMzsP zi$)sKyb|!Var(IESFTI*NMc=msJt0=nC*9N18hh3-o6EN5TEO#c0-S|8~5Jwf$sFV zjzyn!JLvhm74MOXmVg`Ku=BD2H)p|q>}{Z3Eb(X`Xoi*q)2yJoBBv zv(K52IC%JielMS9LvBdGUoG&&xsIM)YRFt@t{{ zTTAhGP*&a#o$|>@FlRVr@2R-w>jTtTo+(9?ljOrr&voE{>?%FGjkyr{l4qDZ{2+As z5Pm!_%ee{Wsjd-y1pjH%S-a6PXC8kPG}h+)S=wpIeSq^(UR~27FZ~JHYL?e?OiS(; z-h)sNrUPj0mHf2Xa{lSbC$_^+yPb^xQ-R;#75*`VIfwsod4E#gpT=AFrZ9a&kW2mM zgEpThQotnYbTqd5L&p-$^at}p&f+_#dp8NvvU*q>X{Fq0s zpDwh)8Og(-;gfBXvNBlL@-{I&c|`cNF8m&0+Sj9aYq_zkRUUOIt`Ad_KZq=j890{k zqJN?7I52(59W@%&YgNVY0U@z}r`!gL6|V4C9X7aVqlx&lALREr&kF z42)HD9$nXhI8Wyi>&Ek*Zmt{C4D^n{!3sq!c-rRt0=ysRyPH6q{d~deV8->;5~TD&C8!GCP{+;uR>E2hQ5aTUxlM@t$l9N)Jf zBBxu7A2-j4?msp0r18yzHE!C2e5U^~L+ADM3tgJdKN>t6_J$#M3`~ezov!>DG-)4f zPUSCv@yX8r@m%8!=UYAYDBJ#Hv5xmwq5T`)$H$h0O~0BHT4=w{_IJ>5?D=%a);9BI z)Wf&n$5q>lQP21g$MJr^W!7zZKL~O9s4rk;zFF`V$2OLHML3QT%WxB-OA(*y>2Z}r z_o1+@raWz{r=SsI*r&CUyjAqer4?~mo!O2DKGTl#{&(&R-qAVBTM!@GtHL?U+lp{9 zXL$#3XeY-p6Rn^b(pdtHntm%z*oq8%McxV3CuD{iTp6?V6oFadT8`uQ~S?63%=3 zK!q)?VSa^kZ~-4Pz)<@67S7RdUd~3M^HQ9L3o@XW!f&_mC_BV`Z@`Dl4ZIKJaFV&m zBg&^f-vo`%VBT}-*P(tHVA;&|e5`N317ywv~@E;&u1pykECk_+dFIduFd-o_o7-k!EfIMan8bTcf`-O7j=|Y}>pgO`u)#bMjZc2eod8|mwkJqqZ%LC-Ry{4D*P zK#zT)j0a)9_rg1`PbG=_NPQm=(-rr5GX1ZEo=m^4FYP1psJG?WH^lG#LLFz%7k#M< z>HSle>@W11cN~)!(^iwG2HAek-%}9gyukr@Gi~IHvg4jo&1>px?={Nma-hh!vV6J4 zy-KPl^Igk8F?pbi=HGmT`BR&WGOEc@l!b?ONBA~lcgGa+RFY!@PsQ=5?|dBKM;QL+ zmhc}h{CR!i_(0ES9|>BP`O)%zj=WEh_X50gax!Bo@`p6zzqTRix>l_}8M1u6OaEuO zv!G$H&TIzBB#Qr!hsBE-S{Pbj?EN$`rg>rn={^33%$i7;kM4v@GWV9(AcC zOF+XXAAi)7xOAKmr*WPtah~SljP^v{rGUjaBQJ59H$E*9r*hjwZadyZFv>`mRm=~y zPitRYkW<9d4LBe2Ft4{6n_OPPvm)^L^V%%GBH5=4o$9|5Vd{Ja-Ws3Z3+m;pDv@)R z$YC5zN3_pazlv~LcQpPU@G}0hB_6hSW#yB$msmcG=lP<8w`-g4J{`o{{7||Y;C!Cw zd2QbBujjSbLS{ir`9D3Yooab8#^V~gNxY;LFs{Qh&+XZ4)_aU9$HpDd`MAj z?b&P%+q2ncg2tZBCXJrWmj9`<*)Ig%p3SCC*|XWbfN5Uo+3eMT6`jrQ1Kjy@=4`gW zLOMNfJey6u?Ah#oggMpQ#|wCz4OEjgfMw5S(*}3uY&LC%YlUhPoFn7dX$D|%ADo9x zb~gJQ$jQ!;a7@Fq+3Q^#j3=JWP8&5{20fRZs#&gH4TAr}7?<7*ejTH59H#BLj@euU zm{}f@_>`7o%QD&#p=JBXaoRA#e7^{9<>xqA$CEmy_x?pS8592E@4PaV?oi!=1^l~W zV0;}NIrq6oqTBYpx#MZYYwdrB&r0GV=krwuin-MSjZyl?%9L@vG-q{%a8|-V9 zbmMFZTEpzOaZEm7Gl;(gie;dIblx-m^C8do&Hld~7rYcW?cYeZg>hqyy_Ih~Tu&xk z$GR8bKhvUPvI`L|;(xjD+cdKu<}XiQm%Kvg^?uNoA(&y069oz#tV zmw-N`2QW=f5iaWL&pBQyac|Q&F<%#Cnf~uackTmooS@U(SG(pM5N+Kp2!uPi>RkG{ z=Qe$w+Qat&oO^NgUM|jsyb7|n1mS*;_%gsUZFc2y!1C?ZRGe=*vifS^^qFhxQIK11 zKdd)bfS>i|I}ozGaGfz%Z&Y%oz9`t*Xsjn!A|CZOSmq30Mcq?AU)mmDM;?@~wsBV@ z%%9pyl!0#(QWhTC3gNaMTvNzXNv;h%7009X;CjGqJ-DHS|3=}@>l4SJ>hrx8v`qIK zEbFJyhM8AdSM)#AqWS)2{Ab?Wh94ix&%9=u8PDx0Iy-Lm zZH&^rP3T-Xf;a2uy%l(-`R#aXo1*165AdijJW1NwfIL#ZMmWWfi_lVqg<6Q)!jFerQ{7{=@zG*((3pqtR?+2WZmg6?V z^T86H4+S3Y*RcGGWPe2HRR0enOr1Z9x5j7vypNU0xnJZk4yJ>7sy3i?stBj`MdSYj z_!$2u@g_aTNM+@dwwG8wGt&Uw~d&zb}-V>GvKJzwb?xro*r0+zcAy|H-gQ`+<|4k9Z4cvOXZo!}@@1 z6S6+wTL9DXK7Bxj)d#!-|Fw@)AMhQ3WqrVR0j56Sd+^`-fUM8f2YfHW>H|8Q@d4il zT-FB^Sn!RQPU{1HfIP+rB;NXf8n!;*eW0;EAZgSGl>aFo@Z-Q+ACNj_eZWrvrg^14 z;HLpA@&O+JJnI7rETq%(Mjw!RSs(B-2xoo3&jFV80cnpr;{(!W{u@5vmmnuQmg2aF zKH!5c4#t!A0dt0JUBR+i2mgTe?#qzL@zuK_Q}+?whcNdPvhCzOS@y21=qpT3zAE~# z?V>Lgak2f=vC%%D)ioftbEMVvgolhA#eoD*U2=o-+u}Y@>ct=B(SJP#|GB23?I-2> zXIkoT<8j(K^NQ(D_AUO; zbZHq>K|@(}c=NHcnD0zK^_iNYvtuP6kJ3#SI#;IP&5rG-0nhZ$z+3ZG%W@vzQ5U2E zG<@>$M?Hy4#~E=tUYsp)?&{)<_C(&@0gG`)Ug9)wd|Dz-<<^VbJ@77qQAWzfO@63- zTKn1?a*BAK1UMfp(`~MN{g{295}thnkEdh#70KRT=v4px5T?#g!CT|AWBUV2H-U6Sf5QSe<&=vHGEqksYfa23U5iPP^Ngu{v$$ zzcE%n0&=orb&g%!UAis~#*-eaZ*T0vG5L|;<(T{-@Urc=0OO)KUdFzB;P_~}5^?Fa ziQ~wCGxhIKoKJUq^&uzB=u6#uA7EP|(CxSU0AO}+Cf7+n1Q@>qfn&C~B*ULa|AfH& z+FD)G0=>I>H@fu}-Wzj0!+Ha2_i%S@lfTF1#x%TXcpTfZaRZ6Uc_Z*V7eKX&V9lF) zv4HE>W^V#5?2e?1HoFy9-*&ty!u^FL7$?$i-;FlMefxe(s+$w22duNU7K=>s1xC;^ zzxD<^eLkQrc@ki97P_N%%%5R@GGGwsuWbOZyLT-T1WTq?ZclbyvL9$PP1!y0Ph&iT zwca7Oan{5 z0;%bA&I)<^sw7_n zjNcgYhf1*1T?t(IRg$j*54&j{SnW60#&wfQ@(su9?I&Hfb;1cV>fpyMk{C@S&n24&Fz?d51B%A;niUSKll1(S~OFs>)HW zz3yK#IG+Ce64$zpcDRi-nCV~>t`nji@Jzx&%)1-|oIV|<#pE;#^TW-*U?P)k_pzX5 z+ua2Qw$B__(pS12KaM!FXA%M~<9jh(#9i(9((O9;)Ry5mH$t3$D#;!4pSmb6E0bel z?lWBg-8m+H9^O{>EIo0zmFS@3-kX5W%cvyJm%Jp7am92@j?3Q38q-)!E=1eTN9R!3 zUve(tKG8wr@l?1?Qgl#1e=cb4{EU^oxIkZDmX+jwiI-_%e338xol=e8=VQb@3Hi?A zITdjnhBwRL6#TdO=5>yHeTka~V9d8@8=8LJjl*?*p6j>y>}Ay@Z6-~mE0RS%&e@dV z)IPKNX`WK9otKMwgD+!B{7P&0duv|#Jgz3ojIL>1I#1@7jyk`BjY4o(! zWVOVv_Cb5MbAV~tbxD5-{X5JYV=T|QWKE!-wkN+YS3{WZwRmfstcx6v>bzq%&VeAq z@2gk(vN+cVdLQRG66YY^HV?GC^zJ7;7tJ<;^U=H~n`gDi!}5z|JS7>HI5dw**DHCT zZK>w*i-a~GcdVbgC3Sm01^n|ollzAE574j0I4gEs~40?v%z+h+Q+TKpc~4D_o! ze|r`3cYEK(do5@%W+^zcH79vJaC~xjoZI3Ui+v6cT(Mtd_y&aGXOE8#5ArHScdy+| zfN>By<{+MqbwS3L&QTyQ>mup+eMgR~-+{0j1icahf*@fG25dp_{=-MqY6Nwb9QXi#zNkJN{w&!OQw?VMx@Xy+nN zJD{tT-y-t;{deA1JWb>(jIvt+YaZekG3od;Z-N8njuPwLWW+ZO>CC}zbMOu?S8Qia z!Z6zk#06!>$2U8Sw@2Dl%h-~Eez;6pUN7+cs(32vXaj<$J!RY^FH`VMn}_h(f{Y)< zeCve##@@j`suBH=SxGnH^cdw5zi#H;Mx6Iu+U0YwcVG~ijWkx1<)B5LNE)lj3WPgr zBZIgB71_7JC(+qoHCYK9(r&6h{oZPFhL>UL&z`QH9gnAy-N zY3u=gk>rfZj^ zZ!(%5|9y__)A?_AY@g15hl6y%j!JZ68@tZsN4(pWunmA=zE)>^TsaH@whs9necfs@ z0$7}1Dy$|h{iC3R-I?R4a++SsYz%lbn8Ty}t_k3IRvLi&xDLy$>^=kbN8JDD4cF7L zzuOlu?eoZcAoSi(=v*I%xd7ek&F`0{8V-q%eLa0(rxD7L^m75D51gt%LeF;P8Nf*k z667BNSh_Rtvgmj4UWGPJril!W&7Ci8+5^?|M9`*;b}7%%`n~B3fy)?eh`$LqYqZKk zpS-A}A^ue8*5UBJ#ZEPRqAO`ug2uN6DGC=i@n-{HRNHq$9OEeSF*hs0l4A4a=HMz5 zfT#WcIhBooar}^-v(28*JrsWbYWV%fJ_d8>^dpoy8ii6@B})l1QtAd+)z{~y3FT-2MnL(BrXFf^3^L>lc)z7zZA3)i6moGv*sGqskH-dG5h)LxDU}g$-f+vIeuh-iAFqsdIINq_71Ks|T#x-5>8^^xNkKm>Wf+ zt(cXZkF;TKdU$YS&6E3i*UA>aw`3K8Ewa730JyZx)KxA7ENd?dU?253zx{IH^f|09 zE0m^^fyo@vMc}&-`Oyw$_^0*Ed)*80O5kQAq&~;R2=k}5LHoLiMsS4=og}26M_Zr+?BIh2F!#J1@n@+7qML4Y^8vlF1%lO|b@vx6CE0eU1#4=$# zSBnnbmZR-yUK4NgL+L&UIG-oF*WhZi*WkmD>DTTvZQRNK(@h8Y7FF8Rk3gU7-h)tH zrce8|>^-=uRDbVFz7}}dgOE)i#<6wD>wwShMPUBhy$G!L=;~5uN3Q|Q?nPi&_afYg z|JwiPUWA(fL!X*DJ9-OXx)_vDhaPVOo?b%U*g*AGn z)9yuhJ9*4r1mf*p1P$B02=4-o-HSjP-HRaqQ+pBK4ZPipK%KIC5#9%w=9TV6xEHXZ zy$J6Ioc^j!r@%rwJ#XBLK)viZ^e2Ke=TCeSxJ(;W`4(W=dP>{lH>|!5oIZ04VoZ4y8kO{a-vKXu ziduoad=I$UC=>N5zK<|}Y8SCSRFfY7&WGhoxV6(C74lS)9|xX_<54^PDd5&le^$c( zbK%eH6YW%eicf=>r@d=>x;%e8@!BvSBZ!IMOpcz?Io5EG1ZCvo4a3C+c5Z zYy6A7AhWFJ|9|o@Vhby6?B38R>tlp6GyRmFQT!G1t1Jy1*U^voHE6PahDLm{?_-kR zfF|o}uq>>v!S*2QYy2KC9al0ghSk^jBmQgOq`tN zt*=3yvcAS{pwYZiUt@Q`ihPZFz_Y%Fz(P7bZ}c^&m-RLFfQ+oKF$b`$uR(j<8DE1o z^WX3_o{V_2V;GLzx$k)&;Pk04xBg;pLk(ZX?%YqeFXYibV4FsNX1o4D%Z3qr-m%O( z@L1y;><{_cud@wh_$f?-@eMfNn~yh)2))d-XQ1)8_L6?V0hB3^9WM~BegXg6e(FGk zwY?>7E`HSOAinXSU9Klw$6E*EzwH|jLAZ$jP~p$xY=7YWf_b2&U+{E!A66vOjIY@L zQ77s_`On1vpgUlydl63ca{U_OJVN4pmc|LcAjmT9?N@fvFL)Z{ZO<=2TiQP$Yp?1r zaF463U^u_v8IY5;0<^Q0!vT*zftZJ^F&qh8rY)&F8?bDBqRr7Km=Byj#wVE6=2&lz z0w4W>;Q~832DsTM3-t$%MVLRehgb)y$#Veb!*V6u+UIi%c`C{Afv4hl)IJvgZte4V zCHyA}e_o$xpXv`h6|}6k$I1Hyd7mWjlkv{UG4`6nK~}DQBCpi0nQ#2hv}n80i2ux+ zCj9tVe&#jP%y<^3=T7DegMO`Y%X`tbgk3Z^3 zTsqE((=p~!iL=ecS*=$aB5wy^G0wUjSV2$oc@yfLR}aVf6t{#eeOK)CXt3O3MK)tLFuomI04=?~&)(4;+?u-vWoB40}02?4DJ7&@Kh9MUR z<0)8gP@^vP08B0O`)h_Fi{t;JAWQeu@NB`W0s9$#&jX%q5MfthoP8C3XvLBji4J*O zJU5`@AMTwezqVzR;~(rVGIH3?#W>=f>6XvOUrjE8JU&IZV?;jthS%c1jw4k5W*28X zC$YopA|1U0qh+2CeJkRP@AG2BW8-G{C5+qnK0B~3LaAx*C&uOazbz6E)64(1@7;>9 z<`Ho(#gFm7jBh;jaR}Ejz8T*QBo@aJ*1kMlmx1)!ym^9p&tvPh;GGio}h z2jyP^`k*^ts(TSm^>TeR;=EMi+@^8D#|g4bKXS9&W6J$ftiR%i*l(_e{Z#HO-fHZ9 zn!tCA_yIQOd*JLD;t2kVmXWMuyozbaBi%a2Wx!?HzRKl*W$Qm}WhQ)XdEbC{PL8p`91gOq-$5H zf2Kw2{LT2!ytxHGK9-+(%``KfH>c?AIMCN;rF)Cexi$p6S-<5r;F;#z@zyb<(!}cKS!`up(^%+VQ zF!f>X#((R>ur6C4<{pIAhjBRL!@L`~tPdlwFt%qptq=1)@)#e6c2uhn(yfM%OXE;NoCB1wKqP>|!rQ@n#+4i;%~0O$X$$ZKWTf z{pN$fS-;|H@ef=Zh4Em(nYwu!zSqY+>(_ub4BNu?DnF-iEpTy67m-{Gybd_n@uJ=_ zH$;?gW8#h$z;SkoDRA=+q?2%p2E4pg#2vW6CmF)MEaV4HhNq`Y49Gg1z?*7odU0P& zYwyt7iQcs}uczQ(?zkqz9e6TFd}Iyu#+2NyzP@IhJ>A^ZaGKyHJve(%8^vWM?yeG( z&M;gKXjy+9Mczx!)Rs4{B%Mq5SRRf%!ImI*j$l?Y0X&Xr;g%XbC5F?A1AUtAWi3ri zTP4{Dx)q}X?yM`r=W2Md$8XYb$0=wAHz4jxavor)dy9t$|BtrwfYYm}{{DTQO@jnn zKoAghA&7{Gp@ppjHAyK=ur|=E+1xJw zO?v-*Jl8JrO4S~rhTCJIF+s7yBCacrCtQUY&kEq>3Nu_cxHc8$;#A)~6y>HB=J5KK zpZ5JE6_&$yMZN^>(<`}GfzR`v76`hg^rfXxwxyuuZ%Dc_s?Kkg(quYBe9r z#s`hLgBjETMs&aMyCwb;U-C~vFGHu`yxufQ++aZ7~A zeynF-l>AtH8s`i7u}Y(}8@!EMz6OtIbdE{&{iRCK?Uzx^7I5)N{`KN@-!^NwXOsDD z;IdohOMQp1`BLKf7Qc+~`*(Hcs^X%r=KMa!z~6&bzS#Gy!k%-rP&MmC6S!JA=BEO+PK1h=ckX?oBo}`+*T8xs~g|!#v=18}mQ}j9aL*8AAiM(kX^)J$( zvz~fMkLg z2l7iPb?0&@od=xP;7_Te>7U(#pRwKfLq4p})8nA=ZQ?C>LfVX9K1wP&Tv^jb2M=9>q6s@JNwI$ZsCrINB-xiS{9 zd+JV*6f|}~>9|EH9SR?-bR>4=(9kZ91MY>&zw_VylKu_M@5WzetiY$NB-e1#H*oIk zfD|sB1K!J7Evtl(RcjuZHuOIoD+sz1~HTk~xG%x!p%dKAyIGrjGGe=_lrMC&ik zYY5NMY3&n_a?-YONatPlgEJlXG8<#Lq{;Uyk5rNKJ zovd>{g)|mwp=hXUh1Ygp^^~Q?3vTs%2G2Pi63s43Mj2-|&&CgTf>(BJ1$P%G zzHBZl@nzSzITOA)4qZ~_t`9p3*x;n|mT00HBPMnNe@aI-ZJL@Jo!G7OGCa=NiGZVm z-C;|hUHk2(3}i1Hj}z+-eahRm?1jUvpJFiIJG5D7nzzZ8&2V^sKvEbVEroQ`HER+=V_wm4lvd`)x5FIaGdiR5It4#Qy#9>b*{GORoHu|4c)|81vK-v( zUKwsSFM&sU=TGBxXNpO8%ub88%cs+Um}{Y)IP(^mpLg-)DZ1sJ&P{Jn9j-o4_At3a zv#a(P?b7VF!0}1?cTQr5<<4L`XGF!(BPutYPj)+K4yYMY7n*~+gW_se7j}%T>nm7$ zK--dC&l4Nqr|TRG;Aw*8|D2BO6~KpDGj7b}+>gMc^14=MUq~;}JDvX~UvqY(P4?iU zyc&0eZ*STFfGZzo0xyqm&H`4R(~y6(T<4;Kv+Erlslcj#Bp>S>=w)~0#-3I`C*c&? zEhg~d=1Dxt>+-i?U|38=PH@niw_uyXVPyWQB5H zd9dSCUuXHaw1>D?@jHnGEW%UZL{R7dtp=ttRGql;40Ba_Ha}u5&#G_p!PgPCx)I$P ze)7Tlb?2(X%`@+;%leuNpp{Q>A@>S>QqRIkkMej($Hq{%bvWx+>u^um58oyG!C8E# z&{wNVA0*8r(WQSV?o$K5P zz0+{xS`j!O!Jq3UUoxHx+0mGRZ*P4>&JVZ(eAy4U5?Hx^kZvyJ8^2e9v+LO4S>F`h zl&`P{nX>z95p46~YNIO%YrJaBR~HvIm5;+^^PW@GCL)+j`86aHQQg(vE0J zl(Tf^LBwl1Dcw?duhK%Bv2{@8Hgk=5&?&PA451*k*0VhrTIKs;;r_^Qe{8rvj(bH;((WoaWLaCb zHmR~IKPxX*?$`3Hd_IYvu9RP8tvoB9rxfVY`YE>krv*A!kMLSOS|9gR@XGVkxtot+ z^|u^&OiLqsCN#RL@yB!uSBx{_EY4?#IG^Ltr;g{Pz;#(Y?sIkO*Qvd{4bLlb9#5zGtCRhzKxgTHC1Iuc zI_?&q@9(^v*R+xI+8{@9C?6_Q(*bK!bvSD;7XRzvQ~Yn>E_&IjcJ))#msmgIdtyj~ z*XyW))OXEM5Pmu&sZ42G7=4+xoX>0mDxyoEi39V7C74 zg*>P0-_-Wg^=~gCZ0p|~E?NKfVsLt=lb)p&VEMW*D#>)(V=*S}dfUH|rK zXwvm>qOtXF;kmH>?RDVO^>0d3dHvfPfmvDE`nNX$t6Tr}X5f@zA)f)3=hO4X^>0d7 zy8i7ggv;yS-Uh6^{!MyXoNsE^0yMYS%%n5&^GqlHf6oki7jnuzo%|;4QF6ch(=soU4{N+@}Uo{EGbyjV;dJBIU zO9+bY@W_CT9Y0q}pTeJ4d?kNZd9?EWqId|IzxyS^`dQkejAr&_$>LI32%h>0U#;b7 zWM9j98ji=}{RZ%~p1;|K|N6jRO;4<6^LM`ht?K0~;r{h-|5mtvn|no0ss6>E=b7i! zUsal|{#P(=6y5jv=}PHTrplY*`XSGeF7*?d&i6379|t;DU+|Xw+8=>e+J3^_bj<2$ zIq)=&pFyLm8h=cuaK$(y&U~d`ggAfc;%s)Bu88;7z+#+{S2!yhpO=U;xxWc=f6KiN zCK<(gh~;6rm+HavtxGs{ZO^|1Hp2`u|E;Y5qHR zi!b$S|Jg>)zk(dap?s)JO$V&))ZwgMSp5HnPx1eUyXckQcJ))#msmfJub|gyx%=`^ z-HOiY$LoJ1TW|sqa@|zFc5mX>E<$Ev_5P>stFxl`km~;y6K~l^&FdxcQNI?>t^E#V zo}|;w1AP}*@I$@H4*u!)fS3JEm1*j4s$G};%^v_W-$H(_!sc)OlxOpy%-{Suu(H4T zD`4ht{*LF=-&C7S{mtJKHh~=0BiG z{Y}xBzZsqj{^tLIPyJ1$sqAlN)UB13`I`-3b^c}(c-h|!usolhH~O1OSL$zeAfxPW zE(BKgH>Gn2<8Ml5{u}=0A;equH|2lG-#iqYU3P9%JOAUL*P-APZYwy3yj}|JZz=IE z<99NV%a*>MGPQp4aNr6%KM;Ff@B=B69GA4y>)l*#Gy52Pg($-u?#F@eEh5}l@)N+_ z_g3Z42HH;ni{D=A$UZ|jepjg@`)tm`H&%DC3YLXMdS|S)^{Owo$kTlhLu-Q?&PHo4 zIB}U3cjIQ{v!thuPGIG_k`KO=wW&vdv#YC=OAP`F*~4L!ojVd9l{23zb2eRND?fh? zKV9b2)bBS(A|DK$Su;O5rZbt`0z5@i020$5Zt(9*u4#Iy_CJ3&d48G6{M($OyGAaO zr5i_+X48#Z@O%tEyDa_IAJ*X{LvZu!)UPRgyyTGLR7L@p_>Gp1Wx?muH}!HEi5@;Z zygxn`+1M5L{`fea)Az@>B%J5FreyRU`2=W^_sG@aQ#(SpZUtTWo${@Lm32tDk{| z5v78w`xJSJ#1*Z3zi~#AF6C3-Vm0_*Vx0YISKG^ zMz=Q5xjKcnJSJQNUin|g-P(=S<#OOLEsg8~XmnNMkLeVy7-z)UIAeW?bAyXB>XUc} zfyFo@uW(j2J}(hxa)*N4i@4XpB%`3?NqLxlruwxRIdwc+fa|h)+~?}lk8Lh)!*fZ_ zMlP+p+jZ;Z^*Xg?Kb3 zY*#-;eTns>czQw_yk18=G@TPZEf1sH30&7r9TPr38552p(~q@EW)aKzKOG5|9&$IcSHlDl?Sb01-0L;dd8+lI0lWNcDcygGqjVB#08BcBkR~}CWSU%2D zKGX5!R`Dd`N#WD+q=nP*``DNC3PV=m=^Bb}8x8+%vvs3uzIWB3NAHzS_w=Cql zXS6S^zde=bQX1VJ6V1Wg9-PLFsZFqT-lu`n9{n&na&tnb1LK>f5g6u#?g-p;TVq1m z4L4TIhgN=#@cn_8bgeN1tX$q2uN}&maMsC@ER}a9p2$LulemNGnHrE z*wF00aQ^*J{0_QshXIecV!v`mz?H|FsZ5{xBA@y($+Ro=v!2a2-Je%ksZf+OI_B-Vr89*`OgR!w!hpE_P-?fxYOW@q>p_h@!5ODq77*$yLQPyTjG3C z0BuaV_9)`h`^E>559QI;eU|fBEt|M6rv9DTrd>0w(d|RLGjlH@V=%AV#}T*cwtheU z1yh$@G~b$UIc^K&ypIpO_4?F1G0s*>$LSw)z9$C0HMG;}830?SW-f2dteTnP|LD$E zBBmCD)cXCvC2E(D5)5AH_mh>LFecZtjoF^cv+Sw)c~2v3Iw-oQ@XHu$K3#XNDlWEL z=aVufe+IPjNuSBRLZ6JylrF_@V=JvGRh-Y^r_0)WMMi?NyeSW&dp^8*x`7qaO`nkC zl0M-dVear*@Rmx?+4N@v-&;lmyZ&5o^!IsHYJZ824)J|NGkYF*?a$8%Xb)O9U~`2z zT#RlHlV4A&`N9Iu3w6A*GhHb@9sm%&E#J|20r8gmkrx6h_alD)?kUX|fwSw>N-4-- zmI?s9I_U3`c@b)=+e*^%eFrZl9WUW`GZLe^eG<>6w|X|ceL2sm-oApc>51rG#!q_t zO5M4txTs&Ix37X$dV3xB3Vout6&$I09lEQ_(ro(pI%MVPfv%7q^fAXJ`uK$4qexfH zM|m~Rr7{av!>@6A4fwK-1_C$F{aWzq6ub=ot>)JQE9=M`fR%MbHf%Az#v8%eRnd_H zuwfeyc27!seetoOHR@a5M7rM0&vZt9gO&eVfh%nNOh3=MtPUeC>Vv~kH#sh;n{yX? z|M4!!SwdSrgo*V&p5;fbS3G_v6=i-@wjsn7_Ix>8G5{<#*9`x2K{gD`vw44XD`CBV z$iK?H#~mKw_%8;By&0VL^YN~3FYz#y7JIW+b1JNKVrh0S4-ZEP;TzhE35V~TkbrD?a(9%Pj+tjHuOO7 z{vhNcm5tTk=i-?2B(3d5B5b$D3l~8X&h7H6quk3SkGna#NZrNPK>d?Drb522@VJ7b zZt-5}aqGt~BT74ciN^oi~E-WV{)G0B(A-ODJwCwX{~?>Y$lD0@&v1?W1!omwc7ayzMcS zA9|qvuH-+)%g1z%P6R>Zp_x6l0@rZ>^D}yfcsO|V(X%7k((H>+FpXPgM`H1*yyXkL zgEe8=vm6!PtG00ah)2FabZh-u#3umPepYi>P1f=BL-Gy0fm}bnqcx`u&YVbm74ImU z*|qSk%kf^8_OA>9E5$E8bvzmmT`HdZoP;Jlv(CpMT}YR`k?d~rJPmC9=&W|+yM5Wd ziv2!dqnRtw^o~x9If_solB2W6hIiu=?P_f|S-tZ!oG~}&(j=M(*3iUn)HmH;^lPBW zwIKLcldQ=*ZvkhJ1tY3-sUJRX;qDQyWx40&;!^os3T|-1b)UJ772h=d#5B0X7zBgq z%Vb`8slz*i=xHSX8fanz_i(M5z7}{pgQNKMF8qbil#Gh256ypk09?_i_<4r6fh+g@ zZwEFqvVF(I+{hTmWsHnXOwCH8^{(3ECZ^alKXSzmXIF|uuDu`Ccn30cm3;`?L#lDq z1*|)GXP|o*_b)6Ay{6nu|A=?#$Bpj>uW{o^$gCg#d4^3bx{%1Pppzkg?7hhFKBRx% zhaYC+o%e$?UqE#4<0rrI1G;lT=lIR<{19;Y7RLW!!gc%~3H()D>Sy@buO8&-Sg#A67a(O2PzqbrrOf^Z$}SArbdyCyotYd`S_7W2Iz`&!_!{j1Bt>!)+= zs&RB=-zdeQ`LF%+EXc0UbB`SQ)|a|L`@ARl-?pW|?a3anoC7gBs+m9Oz7;97sA z81a8}*w<{&Z0eV@ZX22nqnEo>c|VSQy}*V>XYhVy^PAb%EzBE`-GN)!e7^v9r(Mt&9Gd+AxZZK-^bYUSHM1Xj7^iMb&1`RN_uanp3*tGbgEQkh^vuo4&v5709+LeCx$CEP!nYGQVdApUvCA{9dy$_r8%`1R-i==m zqrP;O(@hJ>aDCI_#*dK|#$v8YmG_?mEBAxH0Cpf}3;q&1UFE)x^PaMW+2+kd#H&8* zB*xCa3beoG-j!-zO`mlzXA3GL+w{*)d(v|l7ypL1e#?&uNX~nzn}_r*fOB13dBmgy zNO!mEhklO?>p%6J_Md+sY_>qS8~CaJ{G;w%>OTdy{`1c~r~T(&2-ordHS)XjjvgEK z9jiY}g8Kj9k=%cC zxBf`|#4vO8X+FGcEc~q$hsWtGtakBAX2ccyL)qtuE84sjfzOpYc}dT+FqsZlvd@xd z_IWXBRM=!#-6=n1oYjlj=Oyr}ybs~-@{bM9WhFLPXTr$_Yfi;%@IQ&OsDa)N{s%br z#h>6ap3Z~#LyI(4Y!Dwb?%1Be9Xg`Nk1d-K?&b_S2oJYr#@)fq3~=1YYSYxF)>T@Y z*O4vo^mRiYZ06!@A$UK6lMP+mSVTD35NkHh?jH)gXh)7}ECr^kY#+=QR-3#TwqaSI zJB+*e!X~_~O=`~TaA?)GhmkM+mVdL1`oD)eL$Pvh1wBXq;L4TiLELZ~v70T>?H&ec z?AR-{*CUW^?e$2Wt=&l1ggc6#F7rVZzu;yg^qks=V+dQ@5bkJxvJtn?olE?RC$$U5 z5w7DoKJZj=F+FArj)hjbdCPD=p-yJ0T_}ClE^fnfo?c4C(yO@3I7^qci>?soi54gA zBA1o43%!rl*<@3TQLj51%Yl`3T6(vP zGbv66XVOGuq7!yTfz6hPu;p`v>!P1-B)xQ~2GM2=xuL>U%c7ipX(p_7pS8gwN5D&UyeZ%R5^)r2qGVt*wi%a@1c-m&} zRLjFTV>wU5@mM=O3wYXo&ThkhPT;SmC$=A3dwCkP%J&)JerCAeIo!|XUXhcurwR^P z6`Mg><@O^yD=${=cjH-Qb3Q*^DZk3v+Uh+DbZPx~+hlazfzH(<@mW1u|9(&K%5x8Q z8{=F3scpryFwTZXm-1PRKc-W-Vw@3YarT8c*SI*Fou)(LT?Z`28F_`XvhjI|IFs8S z^|;T~qhJ4cQ5&9(Igh7P`PIqZ6zDAd!-SRQ z&Dnw#&*I8OPU1zxknsl9|Xl$Kj zcrL87yaar@&QfVAue00+%*x8vS>6X&-8#$r0@oZwdLBlA<@xlyah;{om9DcKAzWT( zISQ=2&QiKqd}q|Az1_pVPm#{ZmouHvJiXon=pB#wWd_rEw3Z3Zky|q{%8OXqfacs= z$SM0-@^iFrZ--*YlG!(fC-`N}W@dA}Ra_jWD^21_`4wvS)%$Mf3j7#%d#@YOe`IQe0`FK#c z?+y0{hx;|$({C*_8sSW9#!z`al0M*(VZDqWLun3Sakd8;Rn2tmD!Uro-YOElh1o;E zr8Y=wbsq{$?_hPF@+cqAQrmnOuyWf}`_MN;9uCf~V;j|oG^XcBrP}s!Fneq@X;+>5 z8=IaE@6H}UIv&aIP9&y&e`lNlw|Aj&zCqFhG_>1CBg^|*%}zb1?eDRKO(%tW48M$V z(&KdJQu`C!+TY`OPTSuT2-opHG4NM$u^riXe;>3O?_aAsaXzU|W~rShP1a7H%5$D} zQfg^e+-01l%lU=G`837J{pl7bx|+*Mbaf`^s@_4{JlK=rsn&$=1D*mdnX|TPa&u14 z0GH~j^zxa&%6h4~UxHpf3tWCRsxh{!d-V~|hF9}V{RohKQQnqOMrx<}-qmv8OmDhz z^Yg%|OzYR%O0Rk*#z*~T;ye^|>;;mW^bHEXke{wN_G@$sZo2g1gh%sW!o8%#WBOBv zQ+@>BheY$gUrspXacMxGUlHVrNAW73#`CH+Jl6%Ds=Syd)8SV_D?NI3xWA@OW=V&Y zcBMn|U(d6`zRY5J>u{DX)8RM3r?}t9-K7;B&SfP!{Kznut9NbE;Uln{FM?;|{G_Tg z_%92ymjcrqtv@%lk-dzt&Z_5Kj#B_ztD$e~3CBn0Cj8nShu+w7pVDTj4<6EN>rCwK zfzX|GfOKacAl*4R-N4kOTr4-|EFVqtXs;#RxemLsSloCWu)PuJ+UlEtmD{Sy>_FZ# zy%{=P<#wIEXHr``f;Rq^K>JqiT}jH+w(}&>*8V^HnOwvf>Sn@N?QO*Mc78Hx^=mZ5 zwqbOV&eMOfJ_(bS`nkKfLq4H4=Q2ljyIcj?IGEkkf!gLfNsqOOck}!%es-B1k&bKp zUWbzm!BbnL@cSgEnYLR$M$d7jwyAV13vE;Hrz2fFbN;dP30klD9%yMV=-blly*#I9 z2E3nep6T-oB|`gaW*-2boEK0a(EHL4f-j#F@F8Gjol;%soPZC5vnxF(pdi>gGI_G< z>f+pX^STnWW(L!6CCgTw7ZWOx>D{1LA0Z7N<)@_(R;OuwsBAy3bk@{Ij!Wvp>O03# zFXo$?4ez7>aJCPzqVpGLx`k((*A<=mnv~Y1g~WU|&{lCpy^-$4I9S7hJYC0UjUyLw zw>2voJL*|ingO|vi31^%?FF#mtm7T`kf~W6SRzNMkerVKviOSImb_B2ut=&^Id)Bu`FZbV_ z=?1QEM(I_{Ce~R@I$Ph-_y%doGlSkL@4Vk@ep6`-*RjP?&&kau*X3PhdOb2!rl%vv z`B^;6&&n>16i_`YP>leW|_= zBONQw!$hT&%@B?BBQIdt5oAgCXZ6J{A$W%-*temZbsC}4pLJ_Wq(8H)q?`#$e#l4T zJJ6MFr24o{zIpduaCR*zl&;lm-Y-#qi;npo^ksYI{l{kKgE@m={lqEAu6vKgtdNBE z%U z3Vq4OiY||bRPA+jS;Ge|oi`}+nQeByF*1K0((s#b|7~4bFbofE8(+-V8D=Z9fj&NU znfZN_{cA=B`ubM)t#;vN*0X+V&n52LZ>zAQq|e#WKM=1f0ta=PLy_E?s|{K;MGb8| zrw&7vp^LmDJ-Ba%+IQR}GOX$OBXR$UpU!i#wSih6qMvZqKg9Cy$o>Lcm-^`HeR_85sCy|-1Gp}=AHl8tbO2A=&w@7m3j@Ekhg5btS60kN zRp%pL)K8lm{5RoO)Zxf_`_00=Gu)5hUYHx~jD2CAkEAcGn;Se788zDJ`o5*$ zigSZrbu~A5B)D`8r+(@vVCA{Nn*%HN8EPY%8$24EUB`CTq(WA{ZgMcR5%nX-AVW6b zZV0O17wq5W%}>sb@0@IH?_!sZ4V?U%hyJ_|ITl%IdsFy0rLnZeL4ByTk6Vg|kom$V z5Z2H1F3M7C$dySz;;58tEBc^Zz#;ynp?+Af#3;XgU>SJM;Qm+9N_(5gM% zI^0hT_uGd1Dcmb^lJ-==A*Q^#u;(;e*EkZ=Q%FUW~b?j zc+UkE=UjP94tLg~h)PUd7)Z z;#t9SyZR~WOROKoV|C~CI_jb2L-@2jjBX(0{>8D}M!tLtQ%6O?GI~815uTV7OVT^Wr@a44%D(7^q zg4%j{t->9ErQ+#KVCA(6(!m2$U%ysC zI`iLHt1wKwWuHgBiPkD?0%w<V3FZ=u3Ws z>}?(osY?CiGS0^8X1gy9@r-g;-=VyipP_M`nCPZVgY2&hBDWw3FY+R3=_dzqX~b)5y;yeZq~p{T^A-%5T3My^`Nvi6r1v{--wB z*IS6lzmwfbJf*hZ*afVt|I*pR@cGBV*|nmP##pbj2lH;mEswT&f>d;vFSs9ZU(QeO zW>m)-zgr#e1|H`LO7lU~agIys_?MxMe~BDTOIY3hiZJ~Z0RLX>cZ98s?r?YCm;7El z)6?n!oIc#3@b>MiTNC5E$CGme1W#?g!sR;epG-3w<#Dcr&OAtwZi&35^RLKqoB7AD zcHe50Uj7}J^lkJ?mjRcy0CS`pT1~*YR;Ye-E`w;*-)d?+Agg{IkvN? zUnyRG!KvW$%(EJBoNw5B{cO7SbzX_zaMI0PM@XMhaq_X<&2#%R2gZTlNEtLHiQDQX z^j(c9U}gQ)T;wg#-)V4mO&{TMA%m?4XJcin?+=Y^HGgfsF&)k!b1EO6;g(L$!YkWl zUe(^lxFXKx*fh7MpT-*1v~^?;PVy@n$+FARV6wi-KD3i)=o9=ZH16GuU`w)H{wR#_Zs#PZI1rDh(t?<(KpY#UyE;kKv^|cWmR6{3Jep^-*N^8} zWnb1o&4)=h+!&2DIIiv~tGu2P`+YTI581pckYk0b@Rfx38o>3Z9oZgW!_%WzhId`r z0|~Q_Xnt~%6&(KkSN0%aTSv#|*ssQpWxuCxFR;}u?c()zS>X%ECnkdUO*1Y39o=hy zYgZC}tKIsZpE0J9rYdBln4K9VFGYyqRyYoV> zb+{{9V{~w&SjV+`23u>A2avaH>=p;yuIoFwHEiY4?mRwg^a7z=7V174;O6aTzgFlZl@kf{W zIG=A}BO)HzpRMQ}_NVWvmMh*IhKs%~~x^tx9=)Q8&f>xn})%%1mQ zsm$MdBRFffqI&~B&2PL(cdk0zIpOSK**)=?-Fqv~sl9j`;X3}ex0N03#Ytfw#G9d& z-F-{Aza!k=$-N?*l(bDZI7@dOU)wancV5V^rRm)v4etr}_lEoXxXbVABE9mvVD)|% zk9paaZ5(zi;^O+hXXEqgY=qNY+!{YG$litgSl{PYG(t4*hbC=PYL6cPR(>D(L126L zC(PN5j>dN48Phr!v^+1Q^jHKBW+94YE`Rc+!U$W)tn7=le-rBn6qG}WXy z!_w@JwbTg0Yw&I0HOe*s$cyI&6Xuhhv*bkowVbV&Z!p;umfnZ@+h z;VfO&=YJ!_`Av%xozG<@efMt{d7V%D?oYx~?z=w&EbY5LOW4nn`+iB`o{8O@(qwbz zjZ83gcRDA@`ftJf9yOhF>S=sBV zQ7-LR2K_s=t=56g;hA9U~3#QbhQ z@8sLE=9e^~WMyys>zJF7c{O;?#0-!W2w#e%Ox_9{FLy{iMWWK2ROb;!XXqpTnQ>xHRC0{UXQ}kK#@J zuwS*|`E}r_%8Pk2KkS##$`AWZxc|0JX2}mzp3D!sfoFw%nZ@)f?lR8OWq#Nn;8Wax ztvQ}n$l-B?J(%`^irahUgf2XvviqFJ3PdBGm8_Omdi?P+Dy&<OZXQbd{1)EB-x3*i#rE9j6x{5|t#~#YWIVSHe36H>)PYWI zTRdsoK9R851JQNy)77 z<6DrO65^}kswDIY z^!5yltm@;|yN=Cfv`gWdS&CcTKfGyU|EkTKAPDEUdHm+#{(-e4L%r*auhYZV_V#8@ zPL~gF7~azB_OY?s-^IDHj|CWkRqX9`Y2RDptb`&mMS!W`Dh>!Y&d}m#~gRA?t_H5kPKe%yZYkz;p zZ`QqbV_)CEy8iX+hXvkj<#3J4-lu8v>YkB-E#y4NS^zGTWB)+!`pv8RwD@FXc+={> zjT;mG1)l$+O(Pp&wsZ2dzQoBTF#{CU@JP?_2#kXpheup-hV%3^$L{F1tb$u*sdz)V z4{xHFdq;W(de&bu+#eW?Ce+2?##Lap_H5}J85$m5A7z9(S>4~Wc3^NAIm08pgX<}l zp3R#ElXQl2%rdNW^#YkLQ`5DB{=d)D@;9JjEZ{Q1d_$=t^DtJ|`Qvy?@XdO6s#eKZ!Dv+t z>fbHwbts1YdK1r<(y2+NVJBNHGLh9gG3wson(WPdWpEGvf`$7zJ!O>1A|3W$#`(0; z0#kYP1zSgFX2z#xwvLZW52CEKEZLb{#}b}dP@9XmUYek5feC$>Ii2mF9G=_Gv`!k& zMRRI#Q8saEJrQ3taPvR2r$5}F{_NJ-@y)w}vui>7 zR_ff7f{apsrgw2Gk)<|u4g%Gt)Tf!xem>9YSIa($`2{~@j3b{weV=}6dun&FFJ6${ zgRm}JQz-fdzqG&i6wRzV(5AR@+}6Hhx*FLv;hQPKUBYvWzleJcYn>urZ2Ku+IX;z% zc;pX>Zf(e4nXWEf-;wjlC7klyE^d`c9nQv*yYbOQqhB?ij%x3luS?f4KqtJdQ{ zxPGgV^$qx-`5PNK5!UzV_j3E&{W&26$caW5BhPsk!+V%`-W|?vQ0oi3-Rp5h1(SXa zB2)VH+E5Pi`;=F^%uiCjR_q7&lHlVx|CYLV5p}6cd_`O{yU%eEXEKfNLy4|fS`2qx zs4vs+Yl%br_N&IVAln-9VrRFMdBj`gTbNyv^TmAobnyL_#P2PqKh1YhZ;EA7&YSU> zPAQ#Q2fRF_uSn0HjgY>yER4Q(43I6zMgx!9%pmgBW~|+hfqQLdo&Gkn?Sb#soPWd6 zKGV4sR_5tlh7B@f7dd4=3fC%n_C|j|}&37=kx<)U~+zT-wnBd?&35-$k0zcBFG=ytAaIY8x4c zUTx%?@SZ{$FuciBUTXKkrDdRJvjcjLc4J9)IkdX0j#ZAPkEss&vg^op2YTU(vLeo8 zH9EbnNM5xbIdw)8FbN~Md;L~;cTCb+^kk9Maw1>}z;VTO_ zi`~ZwomghU41gzcK^(Q-AaqYU= zOefzrc?vkYt}0mfV&NroNVRQ7KG9Q=C42u@1lGUD=$V`1Gbt_C-xj<*OpWmb3GTl&TjVa={m9(5Z3qfY#jSS!i-pF zT5dUmZx4d4ZWUoK0>*wtH+IwW@cd=yTr+zy&(h&7qZ9LrLv?uw`y5{4;KGsb`V>k{ zIPE3YrS#&Z@JTPu;XRA$R(+P)g_rZJa2G$7xXtfc{X|@`tYsH+Tw)haTGrv#o?g!x zzB(&TXQOEjTHnyK(^%cc6<3x^vlEEdwfht~toFV*yGmveP?fj9))z2RVzQX*GhtansfBwkR)XyK)$5)&mO5!NY1xdGi(G$JJ+sN6Hey{t|Y&9_1 zq6p(0OJKAy(>7;s)_BOH^)+%iR*i@SKkdbe3W*xo?1ws&AK-@|`_ zmqjbru1Wef7OQD)?ZOa8@48?Bw6r3~3An*4CtEWfyPkPiuYVih>$_^2-7QvRtQH$< zxr#P>q4|gUxeCcxnq35*dASM+k_PxzAfH+HbQ^sdZ-j1PHVobBDfc#+ye-T&#qe1^ zPkbR1nC;;<*cMcxp z_Kh?nzGX?T8fkh5`3Ae&zbE*;p%1ZGBUQRweA}wH+H~CqI_bAE;N{#$px07gWBdA;1(%0_l+9e(qq6n=(<%&qNihpCr!SskrZC!thc zThNhClyqj)rA4j9FUNIdv=e+ECl`3FqZLd|ZXm8vrhZ)?H4fg5`9uCD2>Z1^wzsK~ zO&W~a3k^5_dalwi1zz^SPk6`Xk6z}Ti*G-T?EcVhni-v(-O-v!#ks79G|WQd%W?O# zYy1H|Z|QIf8U1kWk-ZX}_s+x!6R!=$m>t1lI8NbS%`78dt;N07lmLE!+9njb) zSc!c6W8;Wj_;3N|^xfcBC2;!ygW(=s#pNZS{e?#N%!ICga(aHw6}}(mXbkhp1n)B$ zZKKI}2Q+vG-u#K*H6AZR;j*s&%ODNxRX6^-j|w>DcW<89?HLrnA;~IUh zQ{$Rj@w|*NO#RtT^fEh8-Ox%kP%1#e8kN^e2Je1t{G;$2_-U+bXNy*F$()eGF*eHi zlCjYRj4$Pvyb*bYxPyQGCg3_VAxS27t)uZ~;N|sE$|Ea0nr~6s!o?hPfw?Y^H@XjA zk!v5k?&1CTTZ!Xs{B(xI8N|yh5^uQEODo2l>bkkTJQT|G9q^`Qs_;Ac>9X;=#{7cY znE%}gkLJ>ZdryhS>fyj}@mpZhl@l0~N>^;IQ{N5$0P@x5s&YHB4~DpHu2^YOd(uzO zRh|XehXaq|QyMkqCZF#6;~ym~9^HlW<n`PY_PyaXjedCj(Cv7t7b? zHa`We)?0qMPF7iuNk>dul;z?<$8%hw72YU z3W>VAf4MoiEBE)4UN^W<9>YF9VI+Lv%uV%W8byF+tK(O zumkZuJ`bI)a=)4Gn^IqPTh_2chaI1(ZGx;uG zBCap5nQJVmFrWTgy<_#~#X?CA=wQtKaw#+t6x$|2y0(^fi4;9uGX`2Z&F7YQ#0N zD|o-AtNxuvEF?FttV%0_C?Q`iKO|kgEJC>?{qc{JG>h&>{FMKn=*}hoL~tv=pYfdf zG{XJd(KfP|ChIITPabhGpH?@j1LgTw@Yc}@r}qpQ)K)u1Q?`Yn z-&dPE7JQz$kVd5ke_>>pJ;*+t{gcq-NBYUWETcaDjeNUKEthB;`?9j#K3cc%c%U`s zKTB3lF(WMBAN)sgcEz!*voC0ajW?8bm6`6!pW0Uw`e-K?WF3Tc8IRTt+!)WoF2}++ z$bUO{0b#Y_McmCcC?0FiF+RQm23(hHnfeoJ!^?q*?$9n_V;i>h5-#0gjaYf@m)dHK zV^OxOOA=z9QeKsX(pBVHkR9G7j(D#BVO>5?H{)KVZ)PWy*6t~Oe>Jo7Lf>O$s`yp@ ziu*6*p;|5rvZJ8U<#5o({4n25o#Z$kQk?Z0Do@#a--rA0;w`%B(%#691+GhRX|00f zmCHTNa~WrKtU9a8Xk;gJf#5o=_B>7|W?P)GAkSoTc&3Xv4Lhs##}p6O+!Z6Utub#W zJZynr^8)G&zlC^zS@Sksj@bQMlGV!{sA*%G)zrz%2YOB>M`^0*pGCuSepnarp2+Vl z#H@0^HP0${TX(5)QGMHQIrwThEzC{^u8aAVqpR~HvqU+H!E^;>0ZK}3zvJj>_!N)! zkXhNO-7+&bIu?-mEEbp?dr@-d^N7R{j&#EezP1V z+1kGC%Gt9xHiHtxuBtD(J@KE$@Ao7^Wq1nDDnpxFQQl4mHb*(C>=cfA+L7H6oUSS^ z+74UCW%VK24tSy*&MWSc6rG2rIwP9C%(`RuW1d6^SbCtqg7uCs<=Gk42$o=cgD&h`0Zk6*nlF9Ee+~h-@#k2WP=klyF zPwmuQfGMr|sl3W|@;vam(s~lDbT2JmHx{5*SlI=FOE}S~?EO{pc>~V#MH^*lQyp47 z)|bh?NuK9neLrDcX*#^^iZ*Fu?Odck@~-Fyb)A;G!tR>@Lw(gj0yT_2OMiR1D9dCH zSm9Ot>EbCUB16u8}1i``#`u4a<9m#8Q=9l=klDz zIP!9HX>N87L!)vNJbhNatPVEw+{14RKV3E^5Rb}T`B!=_=DCNTrz_T#WEmgdl|7jln%jFZdMMv2-oq91J`BsxG&+kysMsP z0=O=dt@^8zJqcWwrGIyq&(joli%+^@a_$ejPRUWNw&E5j_RkFno>>@3)jjS@a zNhi&xU^)M%-#ThziJf2ZAktL+{!tGwS7*sPhu^ZtQs2`XYM%$@F*~x0z=v-jRWd2t zFPe?el)rDD2LLO72T8hkFyBFv z&S*ZwbVBQT)y^~@;u1%h7){ktiXEnq zZp81AfBBFur15(xxBB!$;)0`abkEqB)-QCU(KMY>$vt=I@emFk{EqR}9_AKJ! znAq&=Bj6GLBe_@TYy6B{7JW{6d>wHy4HnO1LOhQR@yL!x`Y6lj_m$}7yXN$XJ$iDO zXM7a&c^L(re>AYY5$M|e<3z{lcly4N?7-s*>l+AaH=6HKJZ^3+FMSVJdZkb7f!Hv5 zRqxb1O^vqPcmn)ool$+#S2mvr&aTcRb<&bNS>4^)Sv6zrUS1n*bMslO572R*+j5S4 z9_w1EmO#%>C(AMF5TwNWWKC{s-kBg3ZE_5gIn`mU& zb>va~#nTAaRhYN+OS5Nyrw`KSABK0$&v6{;r0g2{xiotwxc*szwLX2dJGWA6kyIy~ zOdq~t!Q29{<2wc0FhAG2%6(H^bxXJ`*3mu-xw9jd4}G6y0r_k;pAXzFcKGU~cf@0Q-PQO`Lpu`8cqZN(us&$6$>D?9OB?vb*!mH#O~HTA1h0k6YZ8zjJWD z$RUJTd@<&o$b`2wSEz95&{Al%24sGB8oig>2~P7d<~tlgc-0IGSI0(K*&&~6X?CRN z4{suRW;w%G2?NI#6mWDE*&yHb8f)>++wmNkDSfJg!O0LOiz-DgxQS-A`qudF|9D{Jbv>yMisDYEAnrzXOK@AY z+;!CiHbCibWVZs=J25re@@B!&-4+;UqVh)A)0{!LXKbuB9m01Kk9K_+lk*`O*_puC ztfnB@0+6PH_i>dSOvi#ceoUyV#SYF-Om2S#_Yi6Nvffzw3~ zjdQ}bD=)z*(eUY$QTHsA)PCS8R(om4-53wYFD`!b8ERb0Z$*6F=g#LRlb z7w|pspI5}YbucS3^bL5SfdwAo8sdudcE`qIDRyWg8X_q6-UdDY_jb)RMQZZ+)A6|6K%f-vD zmz%x|RGot#Zq2Z3S`)oEI(mfD`xdW+FU%iw>e=R0uj4tL`*<~B^D9L6Dt;Mrg0In? zs}5)WsPiEkoK?`}ecRV@uh1`veu&K@j#18&=tWAaxa9b}vCBO;-&<)l|490v^u3v% za+97HBELFMJGdo|rYPU#iyekX1zi356nwzBsj;bvl`A*Q?$nASSG3`|+prWe=QP#T zhf%Ct*>AfXR5un#C7_4+cAf7aJytjG`vYxNPFjZk9qRj_RXKkk+&{>@j?dPXoW=Wloe{w-?J!5V4Gs+S z^|~+Hk#B!?gy1>`Z1b?%q+lJ6c4OZ~oBM`0ja)Fe3SAa_frAeW3N*qu75mrggcg6^ z#6pLQw2Le}vaxS?^ZHFDdvQVbhMs}`HALcNEOEFZ?2rQe8qUA)w1+x)3@>$ICCtZ1 zRyS}^#_(`BBV(Dv$NZ1<^j>IbIxHL5ykTSm#2hNd{?Jv-T^&XxQncAe_!qS)D3f+~ zN+Y+PlK@hsa^LSe2KKCX<+kZ_wry`sPt7sBsXzPLhCNQ@wLaQCME~xiq~T-yZjU_G zwZc)?V!IYj^8oE|MIDWNn&%Xcbl%?AC=ULd-z>S(InjJtz-CAG8QrBIT_%BKi3{u~ER>s1Ai62pCU)G&V_DOKFPhaKP?6LTT`x-x8 z_Wr_jp$<1m9qAiBs!vu_QLY`?S9msCtMJzet8Q57<-SuX*f%^(_BedIvfxy;Jfoz4p&rf~#w@8aF z)t}&2e?R0otv}&@#7~#iU)rt~(5^14sk0w+xjM77;S+acKL(c8n|!~Y08*Xl*bk7%EE@-4}XDI`fwcrR3EM^OZxCP@cYw;zk}CR|8Bm} zeg~#VQO^~l;}hsM`A;eYW<=K-~brd((L2MudACh%p1tdDAf>i}kL zCEC;%3lT`J{Ex8RD-0HcBBaaT>tPW`tDgc3=1Q@h$tcp#jqpZYg9CvNIG<}&y_w*XFk zTjAvM=d!Z_SE+8{jtzVqTf2i(?p@z|oQJLBz}B$$EYbz1KFH_mMrNC+ zB*6K-jZ9^r_j^N@hy;F^;B}D(n&U~U)Lh&>x}0eEn7e53-WOWk&(a;#+1!;@#bx$IV>G4X z&is@blP@|qF9WQ8uZ~|~g4M2DkO!Ev2)q5a63$EFQ#usBtKy?P(lvM+s5dLmyNQRm ztzOS3te@pO%4lYHmn<%&S@5(T?pe#j9C6Oma6DEItAMBV(A$Q8b>Oe2C+e}S!CMKf z@_mnR?+N$5a9_i{A}5(Ytl*GkHeGpkfj8_~}Yzsoa%+ z#W`4@E7q0bF}jNaovTxLtxoM-`w)2Le!7Yz++lCTLv0k)%asNg)7DxaTe#6 z5a(7GXS37vNxYW;i*ZI?;jC&)UJMt`V#9$d>4i^c)gB#XgVjn=&XK>?*71a-PARBk4)C!%_1|k zkpJ_AcV=<%AJpg15r28TU0y%Qd+6)Kdb_oy^lDB{e%CtSVVzwt^&Z|2yu7wf<(sap zQya%EDXgs<0A}l`Bwyjau(ocPXL~ngYwI=x!(S|{t-BbQt*zU}bGo)p?K53lcOSxf z^{luYE_n}qUvTBMbpe*oi7KDz+PYEkBx~!0PuJF2I9*$}6Pk2wooH-rU3f06t-Bxi zbZwo|R9;)R8<>@qt*x5`R=2ip3OHrho30@YusolhH?FNyy3)0E(}c@w>t=wJ*Vakz z4rXngbmqUYw(bh#l;2CrZ_(PiD_tClN4|F2D&1g=J(>A}I%`F)4P7NZe#6|%Zpc@< zgWyU3ESo8~^@k7OIj6&J-%<2jl2OK)zvXOUgSETR%4d5JcNZt?M{-$9;&ZV+L%%mIIMSzeID3<_3+<}$ zU|?n4(iq}M&iuUwoL$pc6V3S?=omtF`XS=ycTaBhel}UTXudUXCqrtuR|Mf^`?;^I za+c39MZtF}rzX!Yve*uq9N%4$iB#$J!$_Oy^&@zGxY8Rg^Bbk7>a*)`k|DVHp6YuQ zezfE;nUKmT;7aXF=~zZvvNn;+NZN$fYHBUfBca9KQO`@WNAcW0%jvu0>I0hDV+iND zu9S-Uwr2KNX!f=za_iF`2hA);(9F)c5AKPtu@6{TPc$c~cQKC#XBQ_c-lTe|+dJ92 zzjvKmr)Dna*gxJaZ7b?QeCPQD()UDu48ihx4?dvN0UIAZ85-$>`BYCK?CJzQ)zOpq zX^++BS`(sQx(7MRTadkYKk2oyRCeibG zTUm+c4g9i08`+z6=c?dHvz>indjA$^m9MvQxAwH}gc@Gb_V#viYubm>FJykdxOT7cQ&NTPoqq27r*0r}ltUhRI_O3v?Y4=K4A2R;cN`;Qj)x-+4}>$<&S&yCx1=x`F1z*y8NQvnO2A^OyNX&PPcPLBXSd`tei~Z!)hut56-#VZv-7jSWy`ExeU7kx)^?%|ZDyYb zu1him-v^)Bp)c0*G_o({JPpTV_VFvg)rO4!t8Mtd7Wk{_iS5F_N_llevdzj}LS!T=fa&u{Jc77XKDmTH?XXVT6;&*wLtncyDmC91NEB{K*4+?a# zuQJ(2_oG1P`ZRdU^QJ!pul)a*yOpce<#OOLEsgA_(CDhhAJZvZG0upyxwxN)IDg^d zjQS+rUjd78Mqc5pYUjPDT$h#gK3AuH z9`jFac>bL8cskWzo$S8`I!pgw2rJEh<8JXuSFEo7(MHZcgB-=7e584{_Em?owqx=C zAH0hH-yt6Lf9>k0s4ua86wh};8oXYoAb3JKzUyE`@qt9RfTO{^(Q=A|77#3KLb{tSN$b0n^*lU&*{9X+HX3q`a8lluj+8g zyz1}4mFHChET3;wKGS*CKZ+-rR~0^;SG90Duljdr(s@ZmX%sgOgh1u;%%%apXN|x$*1W@pls>Gc(#6W890TXysGI z+zgyuoegT>0OqdM{yX7Q`!CEt;=9%F|MDq!GdupVTXW!tXG`3ABh5W%9#3}U=Ez+} z_}0t~yFE462z&YJ){apY^U`p}E_b$i2R;b9f86;g$H3#omYt3YpPNYEdQQ*X6t@@OliS+STacb(`Kfx8r=yf7{7m!76h59`-F$kEOU6ZMx!649 zt>D-CKb=pYHCq+gti-&v;s`j=vg>Otyu6Z0u zKlMJ|<7pqSJ`5njxr=bOS-{joPo_uSpc%&;grj3%LzT%d-9N!n5 zKzOf@(DiAzAxsP5I9E^KMfid#KNlnZ<%IiZ`5M3^R@%`l%}xfU{?Wfu%yUG1IWC$w zvMTr~PELr}rEO_;I|qxTKFv$BQ-Q~Rs3W^QVZPhP2WEU8?m$>+bRVqYZQm5%0#zKR zi_hl3l5gYKc^sjHnd9;6j{Ug=Y5F@FcSJ_n?#VvgJUg;^1~|Kxgxaavr$xuy2|Bgq zn?Zapc-zgH*StxrnL4jsoY*Z}s5X8k@mSlFZYq2hzpR;#@%+1MKW=Syo+BPCr0M>h z3F~KVD9UJN=Smir#g&e=&#UEWWOvPZ8ZLciC+-GZ{+_k>^V{&>J@8l46WhDZADs=Y z@_m$<7)M=wt1k0y}m=U?hA zXxXD%JvcHjxM^g~#=bswBySkpc*)3`^*w8A{knFv-N^vAokf<9yPLF?=aTZeOMHTV zgt?^d8_A3H<2}%YIi>nchyHsNbmh4v)lE9LB>PaFTUrY&om*1a=9bp;Y~ur)TN(gX zo?E&In9VJ1<~f~PQrk}FmbMVKxh027=9adCE6*(jSU%^Xe5P|t_YzMsw+9j z&Ml1-F3&Ap4y-)4B)u&9jO{etziX4u{5R&7rjS$ir{oi9ZfV-Zp?Kn&y*A2Snqq3# z0{yz!`%5kZsrzu--+#y(;@%ZB zJFgHAw%dHJD+%joHZ#g-W>-lTm&KL#PgmFSG_nWeJPnsV^SK@bJna+rw&8zp;IF19 z_5ud{ zJ({1cRF=wJ`B!=#TcAt(FfZHa_60gur|_12uE&8_{vXfX`ai2n^`S8>jqHig=&Hsa z(V2T+wc&Yw&g1D+e|55780akhFCeTmzlgiVm-<{UX(Q*QL5|{3 zKB9ic_Em?owqxdp+qX`&oH?C4QE@vwLW}^asD`Vc>&*Rmz~Bb9gxT zvR|e0PyH&{fU;lp7+|SirLg%`PvF^n1@o(}1y=T}o&wDLs%P+=`c-PbsbBR>!sb^w zT;f+f3tZW+3b5SYP(D+?>bc@c{3_v7zska?U-e>WQol+x=2wO1f?xGA@Tp&=G?o3T zR|2!LGQa9o!0P;}>wuU2ssPLL>3O4HrF5l!)vF1Y{i@djEBjT_!-Mgwq%;2wzv_+1 zDf?CS&hAYv4#gAwsP6hR z`#z4&eLT5^c;t=9HEe*Z!K?M2bMPpotLVBtNK%TtgyH9y9exThw@cCmWR zpHX-__}n)g)6EZV9<`C(9$eUjv0t@b3qF zcz4=(8?aK_Z>StcMz-&mm>U_Jo0=IJ?bH67sR(VIDeJA(do3^SEc|Gm+@1_vtTjZ`6CK{Q0e1}K61m+*?8f@ z36I_tiRU9F9`iZsa29Xs-+dhZl*grkvC=1kT=6Jg*%k9`KGlZj(}AZdFXqYi#(xsp zjz*S!Cfq+;Co>tBSlZ3LeF6GdW-+~ryNt7RIUkTXzZl~DlEsNV%w;9^@ZA5$+jYSC zQB?o^{f^`U32@S+34zc=N(e=okPd+~Q|J;d$)!MY7p_2p8j6aDQ9%#^6%iX2P_bhJ zR77mp5V3&RKv5|QhzS4hcjmpBx3jaC-vNL3bHAJ2*JpO#l()09vag2y{_<_$Nx#4R zZNSh5hVL)mj_Zj7%U3LxTEKRPFsw)r2%qP8mY|O z4*E$4o=g8+XzxZkx?*MCPmZa3R?7ye@06*oj3vo^Bi85CeJR%Um%o7Sd+>wX*FD$f zx7Vl7iGDkM&Uo*y_kz#*E3bcr9}nxVgsZ>)#_-TziTiEBqx%nDghq*T5z?^^w{K_Y z8*Rn?!l%BKlSLmP-1_(Lacy~=&G7F(h&=MpPV3))s^R&w@Koi+GEo2iBWUU04@&w_ zPG;iYv|auCVVv{YmvOY0x>KCmrT+al@KN{Qkvd!9-$9n~?~613{cG@~{(T={*1vy; zYghMVTp)N>!O0W3F`zu(q^~{zTGS{!BK031_Az4*CqsI@U}ldOE#n9B2cH&w3Y*>8 z+qq&z*9vP(78iv2c1LMo*Swxdvu982UA}lJ-UnbC@)y+yFJRm8S6utH0|03={sCCp zX0Y5^X)_)H9go4<4F61bx8|cl`xsJbGb+A0l>NcdHe&jm*fu;49sk6SYxT#{0J83COi=2d|&RkB4nL2-mjb--d^60&!0#JnDz7 z;o6FaAcu$L^N{$i5ZeRFvu((8xVAjbcC;aG@B{(mp}n>ZX<}fN<7wl73r7_f%R<|b z259NW8%R1VCo^e7XuGx{BS6nP~TLm$o4riq4HxC)$u8%d{a^XWEdbpfhbl z{sWk8L!QC)F>`0loYJ?bQ};n9Y<~Bd13h>-hiwP1S?gvkJROY($MDm-R}J*L{hrLS`%5_*h6JUoW5JsDEzb5hOwzbqH~?fR;+*amG1oA^Obbp1tse?iVlbL~$CKHj%In?at$S37vF?aLN8 zxBj{%uGJ?=w>f@%X0jDi993NOSM`~fg0>B`TO+N|8$Z?OEH~kO*R^PHNz3;SgWjT9 zY$tk8&+ZkL$?u_uMmIsA58pU(o7v8G+N1YwchI(hb`PYaAB*!K3z&=Cs>zHjb6?4K749kd^w2z5nQ8urV;@Jp9FBHH zT%)*A`Y~NQxpD{WJpmYhV?j%KWu6U+v0{_UFKG6~HEH%kN_z4}9kG0@ELX1JI*f~C zap>4xb#pBn*T7OPr5^~{tWyUe9fdot_5PK(Casn2^Nnw01IL3n=7lzSy^Z2F4o+|o z$9KgoPT#}VZ!KQ{TD6OEv|P$EhZTp5zJsvO-zdmB^hwZz<87oz|L|*WofakfWiksZkgOXc`TZ}#VmF`MQ7Yjb% zU3}H`*5StVb}d=H+S%IZZc z1{QZYQBp3Xd#Z+RG3x@pWu-bt;2j0E+u0Q5In_$HbxN1W*(1ff!DuYwgHycY=L_M{ zmZMXe=zA`%ueO|E=UGBo-UbD(uQBUKRTKAlRuZzjOq|B9| zcUiz=6@1ET;eh*+UCc+f;wNG0HcP_z$131vPdYkyxy7C5aXw2l5xz|u>2H?^PTLuU zUrRXV5B~KdYy@m{eLBHee`KA!#p7LwBQz;%7+~SMN?z|>o@SAEx5Y=}b*8RfiNPq% z{W%)MK3o=zJ&%A6cc`~>_T-7yKbXIwDYyx9XQa3e#m(&*K@GXC*uN$2Ebn(?>hNyl6BK_6^~ZE8Dp zr#Q7s=L5%y&f`@l#$Z8~8H4>0?{#sXw;e#oXXku&!Ru+b9)aufP4PJT68GHJy6a;g zd{?i{^?9R#o6x%ygN^R~K1ISu8EcpexlyK0^xSx-F%LL1_EMyK&T#^8X`Me2u(Td? zzBv;6%AEw99%Hd`q%!Y*5$2mi-bc)T;I$7QR;rkz@2Z^)yHCN7F@$~Ooja^LnR_;R zpLw15W<&qnE`EoS@8UHg+yU2`b=;>yMn+A#q9_%dUAJ8!s+oe*Ul)k|yTE>(FELIq z1M?B}85<|)1U(N+%et4NT_m*XCu|4OIb2tkx2;$ryjHH)?B{trr|&e8E(PsINSC?P z#}w2jPX{d8pTt+-V*zJ;Y$9TeoCj%u!dF@Q)E<$g!SO~h~miI01XM^s(ciJo#1Loi7iqc(s?|isfED!iN#L#}4 zZHOOS__3L`6=8EE(;l!dI|H&(zg_`Y>eq~?ZGyD|D}mEvbhb1Z`^yw0B0H>~-QcHx zamtzB`z%I6t`K~ zo|JvS)TVXNrhV9I5Lo8*Cf{oC}zBDlf0SaUS62IRo6!uW@)I;AtDdd)3N#_f4SVk+vsXhooy9 z*bZ!ovF?RJ`(~u}*+N}AFqCm__BjGRZ?tIDD%UPu1f6fek82z9?b4*4RclyX!pPeK zN^$GAN=sR*aiHp7qI9rLt4~r`R zPvfHR1`Oj9Qw7-u;QRGnzKM%s-xm+_Gig^{Njt2#8t@nGgPp0aW|HiKEUMmNvq z`bLf!mSG&>1#leUVXFb-I&s=b7~=~U0fu>UHom~|#rvQmZ9A?3td+Lo{h))2O54G? z@+LZ0hAs5h&Cq^a3wjA>~z_#i;^1k(5wWgq5dJ)DpAAuZg8`>a`*B`}?ht^+?Z3x%)O>n6WSKVNPT8Nyu5G*?Jn8u56M)(A z%O`QIH(WQqa34dR=R&1!-NJfk%s{}816rnEFmdYCNy|FBu~>=Yhe7yc&(`(P7*k|S zGPul0Z%lDxnJk|h+_2BZnoevYTWfQfE-H~RSPQZvk1?!PpKll$jCQff7U`l6V;646 z^6sw1Y*Rx13@pzBVp|?$#`R#d5tqrI@K5gIj$m_p!}@=g8*%ZlNF#h}+^n4vZ_IKmz`;58EI(t{eUc+66 ze&ZJKYah@CIlR6VKOVN>Q45FX6V|_ztcukFx>&#vLM; zJk)FZjXP_2z9&3Yc`;AgZ+sWD>^HtI=?`);lYWD?Yrk<9&Ux+2EZR%mDNgOue&fgB zqwb#|b+)442(nDSak0E3Wc!WVz?1eH-v-R~8@J<{{f7VYf_v)X+RZ9Fy1M%0U^J#V zD9O`#rJEp2d!=$pIHQg}z8)H*&bKEi{e-^0u9XY%Qjd#;RpmvzQ7nB=y{t%U3@GP! z`RcC4B76;>2;$k6Ex$?5)rg2K7`MpNc{X*sN1`)(x;?GV#66XkjeFF~l?lin+r1yk zef2zU4MgKdfcbXMySsbO0I;;(W4&pm-TN8ncnsF=vCY{U?cUFY_HLxo?$x$ALm5A_ z?%H*HPN-}B{P_77(031hoPXPQiw?%SLToSY1&)3?$as4G?b3GfZ{VZuzaw?FqFoHK zOuP6Pp0)7ZN!u>|7CdRY_&dOCyZ8XE-Q-i+sBj%!-i@mwd=ehmcOA`mA^qY-v0>nuZFoul*b6%0pix?&*X_ z>v;}GzaiZE+JA9vd7KUKwP!^xd8pU=T7idVIi3a|rs1gKVxH93o`X!*_a=Clw&i3d zzDC>C*M@_h*S^f6z0{rJ)Gqb4cF{RPb;8$zEaPkYWMc16L1*e~{{hVU+B3Mea{*pY zSnS`a<9U3y9&7J$U49?m!?>q_d;dfHm2|UK5!TZ?<;;Ojbg0fRH$Wcfm$s6re+~oA z`DY2~{d7a%Qa@$hTJh73z{_JWKc$bn1b#YFXh$LSep)i7)<<5zJ9$t*82G{C;q}v)l#=5s(O!?l ze%gwi{I#DdxiLgrv9pJ%1ewo_EXH`aiPyImy8sr~F}DZ#y$cff2+e-B*Gz&`1S$t!b+-!rBk*WLN}J!ATSaowq`_l$A2#J42z~=$i0~h(cSUxLYz5EhzeyQj3v;4Y85C1aY)4JV~T5hF2t9!*7ZbUKA zH?13=hg#zww1{(bS= z1h!~s@EV?12~SmC%#)4*4+Aa7en&`pWKO2( zQ`B~CM<;?lmRYoyx>KCmrG4Ba(K%UlqTLO$OuPF)X1;X*c+xT8D*>}(z=LsZ=Ucq? ztBE=OYAeR$x>{Qp>uGOH0M3uwY^Cwm+ct*(9VN6!Bb9Mm z$(S&3vryFQp{zAz{!Of-&9nnNH{*j+BY|Ln+}|>n=&>+ z-JY49qoKK6bcWx;Y{FbDSMOfB*Z$+eM{)yl{BeREKYo|7lMo>B{yGbK=&!7YJhwjd zYFulbCvG-=9CyuOiX(@62OC5j^l|d2kI%=s^|j-0o#Q{gR(ABYov~JsZOdHHGMy*s z36h?Ov?3dp)Y=W4+MVO8wF!KeNq*I)lf{NpBz=vfry}Jzd^GIkea35=?p;UPAIjFW z3(gMsZ^Xs#HN9Q@mwhhRC~Ugr2E|E`UsDqtegW{-$Ji&H1{l{8HD1I9e|#(@~iS}Pz;D%`aES>pFbPdmdDu*pI>cwXsh+*b82{A zD?C+su^hDDTmxG6o3E4f^*NcwKh$>GLHXx_o_X=*9PQ2F)GqBe&ljB+s7_RfAj|Zd z6En~KSAi$(H~Rpy{boO|S$p^dVZS+DE3rp`{S=OiQCgY#{PJ!u zDe(A9;^waD>+f30ryG86GS54qyQh22%ASEfHz-&II={yoZFY9_P3XnuaJ`H(tXMOr zYnA(o(1ICrkDCrFZC|)oOC+p(vVhhaM^gPs{!3 z_#^i_^2jRv_(bOT@e`&@oN&SkCyhVtfP)uzF5Yk4Sl$b^VQkYcmU3dRq2GGn3S$AT z1$--T?z3Eefc_}+fabJqUEm>CqVIp02lnZ2gDouMwJ?nP6cxpJO;f-)b@ol49{bkm z)1^;+2l#BC%Iiz;<6-+$!nIF*r{Q70NZe%!kM>PD9Qzo;ZU1-$t}TzV0sZ5?b817TJTZ# z2aq~j(LV-Rrhn|stZ%p!JZb-UIbgPbd>5{H=l%MI$+-W%-LAls=X#2tcXkh$d|#8m z3`bd}@Blx*ySs}YF6-=urRPASt+gl01}r(5unZG$cI$LtysjtzuAT4eB?`@c`_Av4Q< zIVkAc^=YrOtAemrcuM!2?XpENvd*ND3z z;n6lFhhrN>xNX-yjcd!}Y(TsA8IemK>b33KXKQ#qCp=YoF;CjA-2_^;EuWY43ptrd zyGGl!UHdZ5dF{(A+DqLjPVLfm?Pl;%_g9cQThXosS*BgPEfX*M1bEVR?MA?CyY?wu z`#bOBV_(EI>ypCgCtm{0`H2TQKlv(Psh?0^D}M4d(D4||PxgVIaK8R^q5TF@?cHeuiOEc^_B19 z+Qyc7?e+!Nj!?z~fzEWD?0SNL)59Q`^!rUuI*>rji}QGU<1-+CBp) z`@oa>)B}K7pZYznolp5mC>vEb77TDat?gQbwW=#SyBG6aTEu0#7SA16#Nn{SZt}G8 zbySWG^E7@a>fTZ}@15jnbycyY_T%+DJ$3`1+rJp^y1Q|4o<{Q{kr7!_T&A7UeU@aO zZG`3}U8nn3l%upQ@_s~n<6*$GeOUxw;`sIvz|!`G_oR}kpV0F!{PbDXD%iL+c84Y4RbP+c8j)ayEO{*y!K@l z?WOJ%r*>((wXx{jM0KLw3bIVQb!TQQ@GNwu?N$SFY`fLOwbTIkRa>zUuIYcV8u+#H zBLR0lXb|+nO#x5+ka}D3!w&HB7|ah@cR7B3iO_C_)cc_^L+LqU*1Ac$m*VHv;W5RL z!_9%8@zA=n9cUR?-5zO$-qcs(&}I}t56+`sH#=yX`oGeR7CV%#B0rHq+3F8ta2|Ex znUctIGElz*&eZQOpnkKyy$rObRM974Pba|ETA=G++KL?kpNEGEJ&PD;!ul~>PwV4l zm+b9;^<#j|?_NO?w|TB zwl3wMT2IE+#Y^0RBl)V(DJH|RhuH4Wu?K#5HtW_HupCFBhOu4TQ_k5>@SJ_hXz3^R z!Zq(reoQ-I#q!R+utqgpp+56|ah>Tt;MYUz1M4^8TEEBQT?eGYr|EdSe>6MD zV}0TPTx&g}?ET4)_WwYpII6fR$PeFtHB%f_T(n1h ze-3ELKNl(K>H9%e#`gm*cwGEG zhVB=~HR-w^Hf5FR+ltpnKB~AzaUv@<4%QRu*Y@W$vB%v5*jjrJz-!Bf?N7*~X@4G* zwH~eR19L3&*R?-0b2w>#<^iXb*v~=Ri}`@1?ay(5tqp{ntBx#=_i|19GpslPu=;(X zhZQFhk5~PBS7Pe{e#md0|7H#rWbx~NkUOk6$;t51IL?Vr_Ijw@Z(${|xKliQ>cEN> z*mBZ8e>>H~!sc}Pj_d-!{eClH`$gS$rnE+#N{phg*9_OH6Yc~(+w|p^1As`}Oj7WN1_!<#11(dxR$J%_(^x?m@wE zQ4rR%V9lr>_w}Q$J|(z0UHt>S-IMs`1ZV$P;GR*LE6(m*(}yn`o-MooNghe_U!e&R zlL>u7|j-T^$?s0AS=DNkLuI^!-BI;AxH!cLMmA-Ki=peqOzHwLdjf+9gqYLTS z2F=a)KhLvoRDoeXp(W6>6hGZh$n{ld)AIAX>B|@P;@NdD$y}T9@ad8V=0n>*_KAdR zpSS|&wofE(rSL_brsL5*kvz6fT!m|Gx5?i_e)NfFGR0BFMSHYQ>;*0Rgg&ICXB`c) zGJRseW%@+o^w2z5nQ8tm#+=UY?>c?j#068QPdI7;PW+y)%_6`i%t1XQ-3AdhYtF=J za~A+Z_=pHUYR;4?GmoA&eL9dQu>B#vBjS&rFlXkpnMVOTcfpi7b7sw%>zBmg`+x0d zjYl-u4y-{w%$+^-P;N@i0NL>e@NB%nxv|htp4pjknh}HQ2wNHdI|sU7i{DPLL;G;; zznQW;>CdsO+ltpqx!baKbpf?4{T4^Kf5650h2Kk>-zTv${QYVa=NtK~E0m$QKl3A{ zI8eVgLx`-P{W%$LG%_9sPm~esB*z(q>p0`hIJfrJ)@$gs^^N0mgVfKrSPo9br@^-P1N1^;#|ra^m`Y>iu1@TpY!7w?4oLH{0S_ zLWcJ=i<6#L`_5GshnK1GHI$z2&K0h^<+w@d&I6t42g5sYmz4PV%>|pa*pFXo>D=Pg zDjogza?qha4n?m^mbzXh;X`t6HBSJ!Xb z{qxyRj>dk)Zv{P%bx5yo+Dgq?t+_^v>DDKLj_n+(vYTJH{t#_aejdt*bm(;TqBIY^ zMD)`JZ421v60UvjJ8^FNT;eVhzR1&bJlf}y$M(5b;F|3l+d=Zbi~Q(w-^~<96&LN% zKKDw{lK(2C+ExZxnLan*GJP&_dT1Ufb8EbqPF$Kl;#hC$aAP4X=*#pU7N`A(&9m2s z_kUPXNuIkpVtshog3;ba7&JQ?zoKs$h z>$5e=vW>n9wG!7F=#VNS-ifG41G;mPaRO(x&PA%Yk>0`Cv+un-8Wms!sF9 z`5(pZgH%L)tHV z7_e6Qg^z%auH8zkb0^H_J_>ps*CQRW{CUpjRA89TeGGa&j-SrwT$^9puec96 z1%twsnakZE_A?LK9F(z;oSBs#C=-$B2UxtXum=p+pl~U*V>Mf|0eRIU-=wU z993MjNBfn}gO>fu7m$*kbv4M!^eX|E=~sx;L-RnH-0On4G=Fn24ydk&@~_JJZ@2dK z;Df|n&VTidS%)5fTm_F8W_8WfSD{PGmU-j#*JvBcW*I9Yj{Ljy6{!VBA{D5)*WaLA zIW*7gzXMn~zTXD8y@%h#wcaQ4+=?IV{1#Ij`t1+SG|$>+{{Z-qAJlrykJE*IIp9pcyp=pt z`Vb$MqhDCxTOD)$XC`Aqjt;MP`=4)=X8lTD+0bW3`HA@0YmcV_i2 zU%I@T;Wi(u^xx^?^WVVa`YaFU`@2;}XGi0Q(3Q5SKLRXmCs|kde*9g)=`lK!ySlpi zJljcELTu>tnb41+|0nq2aU|#NrF#7{z(?UeYkmDWuKClt6@9Q-+zmJn)-A%!78N{6f-uCH<|W_aUvwF+Nkl zL00M~EUVB?$T{<(e)b2Pvuyr|9}mmVvSyyC=g%d&*uJT3rF%%|TssI}Z3kUH0QnCB z&piJHsoo2{e`5fTwls=|LBpe}KiWxLS!cwl&cBP!e>k1Z4y{AveH5^$Gx8FrW#jV_ zaVqyQk^4B(9E>uUSISSzL+hliQ%`~}$MY26JhZIWx%=p2T2I&T{3q~uI^MsW?Eea# z+W!o$Y4fv4RiE@PE{0a@mY352oCkFWVh9lbEd zF$y+)6MgJbWY3kuaC6U0U-=l`<=b@2Va5F@uev<=IcHn(JJ77HCGvgZ1EAseBYFoG zxo=1?UdVgFJpn0)*K7~)4IWHT2=sh^0&HRzIz)4gk2zxB@MiH>;Iw}``89lY*;f1w zF!b5(V~?|z`2A%b0c>8+D#!CroG&!;=*x>dz^&o^7q0Q`UAH#N;dtTZxNXG~z?Jtk z46wQUh%xmupK&k5w&LI9IlFU}${>C&-g{54CwJi!x5y*&^&Ds>tl}qteV)iO8}_Sw zoKMBGv>>C2dpZTL`}_C!y-n`gfAZ4F90nRKt5-=`4hIbJTdcOlyZX!f*QibHfTIip zn0f^k^67c~j*$G88ErhOx94mxqgjlA41Pm(5ysaF+X%49U7ZSb_nh^y{H?UQ+RgZ4 z)72S{g|trSw?Rh$=8TiE5$DRI0MkrN=w9q2(EeUFHi6!B9K`v;Si}uC1x}BS`r6Oo z(0V-1KR@Y!JdSI|K`4Fh*ErA3SF~MXOrxQ~OD$8m0c zbWzh^>Hjb+JM>U@%26ju&ksxEe5C{)ADA$8aeJw}6nXajN3-K4c(yVMHtE=H8{lS| zxdv&s#;<5Li*1?W(6WfQm|yL)wgWBujO~$fyhvIdx9xy)9qVvTLEMh`@zAjjb+DhQ z!*M=ThofHNoZZkm2|9SN@1$b8(`W_9FBH^{>A%6pVRNbyYCI6Z0K`k zWuBhC$9c(gtmJneh>&)Od@S!RC`&DYKe8T%GH+yrw7lw9mch3uV2ADnm9pPZ}%pT@s)jW&7b;1 zl+i5qr7Rq(YaRG>oU>n+2WtodPs7=#{&E1|w$2_{!+((QSM7;@rSqJ5n?9e(y-eUlZ zIwLP}S~fl}5vOvG6}i)q=3tajs@JAGv`*SOH5+nrJg)|vhu+6^?wxTg%b~ zThsYZxR+-Bvk{)*asG2ii+R(2_E6B1H9UPQc|7wNT~7o7kaJ1-)Q&Wq%{G%q?CcsnnmP3gSoRKT>XbY8Rou-v@pG{Dn& zk-$PeJ#U;B(JnhL>cn+AFIogxIxk{99m>3jb>_vG7cGUHbY8@96X!+CoDS;Y_^uU? zchiOP9BD8Q#kh%Mw&l>nG22||(Kgb~U$nolIRESi-}{qsxXv}$R@?Yml+!F$N`3}= zPDFF3&BAg!rrk?=M(OFIK3wysZF!W@Ecz)6hw8HZ=2=-Dd=ohEG@N}p{#ygM?W@nJ;eW00SM7=YlaBw+ z1TFJDAn9sJUnlA7kyhkn#(yEtkY&ejI%ANUc(kQaycIM&s`{gy#FceMoQ^Nv zCOY5lbT&J*K9Tnlz@pB`OPrRC&r8Iq+)G97JCWvKlu@eVraZKM+WK_`=Jf4pCFDLuGLZ|k>2iLUueMnWG9Sgp{M$WY&hdP*#SU;nG<#6gd zs{cChQvU}<566Oa_orN6;{BnX3&aLruVX#bI!C-M52d>va2_voEO?C>3w|6jeb10- zIsab=fORdg?^8YjyV7xAxNm0Mr*ZGKgXKfwr{{u3#(J5|`WVZ3ph?GfydQRa$2K7y z-@OSiJHF$!j_)qUxsF+Me76p;bbR*?z;t|f8P4taj{a=NcbDT@$9E2A#&_=mE*;+q zER1`YPdmQ5l00U7N4y>1>9rl-T>~0Bz9Wr}@8rBRzWV_1c6>*h((&Dg0MoM4@!f|3 z%Z=|o0(d&U6IjTn=Z)h#+GWRgAH{V#zWW$p>G+QIaVXBn3+{FHec{jKaI)s|i@=3ow`&L5i!T9|uDSd&V15khODQeLubW)uwK!vG50ZKuKx`1(mF=Zcb>sRcsJ+gD)SQH zyrB_xEnxBMp8URyUtjZS!Uy_Mwr)j^lfifPZn1EuAjZD$0uJlwLs9k3>zUNs#ZvV& z`aU3ba9-5I*CD$WAB}I{xM!Yf@EwCrd%-`Nu+ZzEN?vZ7sMfs4y#St!07 zi5m|b^6M4=hmV=L4;*(_1d|@w`zF4Ej(3uF;BWK@tdIKeo41W`kkCiGxt>6wZFO(^s&sMz0{rJ)Gjy9 zhtBVb&hM+v;YOC}!;Z=9pZrzur28d(12D9;@+`1X+=6R2BJsB$b=t#pAu>B0eGi+o zguA`uU6aN*WTX%9*`2-p%i(gaE%*5N{b}vfa&Y-Vyz9fF+_`Y#&K1kgvEP5ZMQoYe z)wRl-Ubbas557#YYF77DeDF&)O%1kAo`^kjoG`z+obHoPnUg2tjmF-ARsF_>iBqTI zQDy&f?6PBI9M{#$&nBn{lh>)sR}ZXmyO+9F82zEo%cM1nRxDo>TQFNb(I&s={vDFP zn78)EcL8(nV#vMXdXFCfmbO2<@2#{yKLj0*bWCUW*=KvP9oBdLNNDduI@;9&Zv@*O z+g=Ri-BI$Hy^)h44iltOpc2`Z~{TJ9RIvwOt|Zm-sPm_baA2$U{B0o%#)~b3DHlo+>Wp zLEEWcgO=^oeN3U}ev!%kG1vi&{a&Tw(mYo5CG8vKY5Vp^oJaXFpOlf}G(XzD{fV~1 z=0A(Av^&bOc|?q8xWSpWZ<8VJ_w#e;svE!Glf$|3JJzZJSFT=c588_d08hv8zb7p0 z<(4jc7XcT5B`nwU=OS>*)!bWOx@M*>3vMz zE8tp2*8iS`r=tYj3&Cy;-V=ZgUdrtM+KVSam(;VQuzhUlDbV2^U2MLFAu_r)-^aH# zo~Ca3uCl~D8l-KdKK=)EJktJTI9t48zFVuaA;z)K2;F~?9=)DiwNiIjN1uf(*3r|U zU)$GFxCi-t!ttd67N#5R$~o%bxa=Aqelek|H;~86Qs2_t-DTbVi}j!Wh4Z@(y03-8j+!*d!MK-&!sgEOu+awQH^!Jg znPVl&7;kXXW=@(jWyX}5^Co!t=NX*YIAhB66a2m!MI-8(G;8t{Y@RV^+N?PXAl=C> znh{^OvBp7&S8pi$YHW(~i`5~st%g<{)RgD>o=X&u7hPD-JD~F=_-!NiTJK$2k9c3X zmX7z%AI;)jXj^zxan(IHEZ_JYBOlj_hWdb-ZP$vHZHfDA==bILjdAYD!8Miq*}CfO z-=Nq=?A;doZw!N4(la0Y)$m53wi5OFq6ksrQjGVx1XY>?yoELvK59Rxf$= zUOs{PRMp*H>?Qncv-rJ-SKylI-Y#u+?1Vh{cCp!U8tmY?+R3uJ0l$4gJC;B2@mJL| zyx0$T9+tPeMc(~|S8W`JYub1KQmyk^)~x%?ch!asii1Rs)_Gf2dXK7n!-|6w{jU`L zhq%=DBP3Dk+z_9m>wzvhyQS`I8r%YS1Ir;h~usgz7X=Jelu0x#L^K!st*7}buwzM=J z<9N4nvOE*s!}q%`xfsC6!sD;D-Y4BQUeN-IxwKG+xi= z1^#yVxHn_gd{*FLx_M8ezbN4RI9=oKS2}pKQ{mTJ__!F8|5A{X-kuoWz9Z1k8vnGx z@AKoIX?!=}`2?}NnGf}|{kt>JR5$X`GT`$EpI>*EdFw*?++V@D4VA*Pkw<_ZEKbi@ zet|f;*$BkN|Jk>5pTBT#$2s5eEC1#GkY)?eco+2dfG|FnsFa#vXTC_sXA>1VJZPyr zpD0y8uZR7Kdp;5OAuR1nbU5l%I%j$nV6DuVjsP8tC7m;+&j2_#IRta3BSFt&Jko<3 z{!Z1+O*meh06LBrIlW|i&X`t1WKfQ_^^D&Wu5*>iIJa{Z;-&~+^v{?_??`#fkrIU9O{ zttgFJTVa!ygVT+EGVqyxQe#toY_$G;113R#T|YTBhm(GCI&dMu(nE7*e^*V+TUnpdYA{u20%zdVMxLa_zn2QFXh_m`7FSLZLM=5XRKi-FTSOMmIY zHTOu?F#*SEOFVzcCzb)0`owa;TJed~LD#BJjD$~|0eT)QkRHsvV>9*edBzn)SoDdN z(9w;b`h=^qwLURvZZO54!Xx5ymNUg}+OGGUK0&zpL?6zrPY~BHe37T=c+@AzV}0Um zTx&fh|5@aRPpoE&ql%06s86f`Eq&q~q@*8<^B^k|3k$f6zYwQ~=AkOHQQ5;Z$VBPb zc*-YV>TOBO#pcW7%JB!ygW}HTA^;p!z8KSBJ4id!uiu1oYgcWVLaxm#=Zk^Q`1Qp@ z+}Exb`s@69Acqsbel2iPJ2CHRFJ6c1bguY%z}5;8{d}#LXZ(CvaV}tk?MZi@!^b`8 z&i69#MKG?(>2=S1F9585KGi7R0KA>Q@j3Auy_{tJ7T{A>_xE<<@rtaw@%w{w-VzkM z@v?_=?xJv=z33itoVx^^TcgBrzTb13^B2X1byDze-7k*wm^m7`lpmGuBY-e zx1=<25m0F_bi937s-JJp&;&Q(fuZb48i1I|+vkMhP23*Gp#UBq*~ zQa{&exH$*cJ7J2UVoLXvc5q&@OPR)~;=H}BaUtqM+TXkxuvYq;i$K?Ef5W})-U50a z7bD%Hk#FOVsQZ@Byq=yFeWy%y^;r_LH?$WfG3>WO{yO|PCt*h=Te;8euubs3SZ>Gx7&q|_L0Z-JMY4^wxy)I z96$CuS1`p<#YKCx-+4D^+3#G5l=N(~gRD%y6L6V+hd4bn4_0QHKjIk24>FT}M{V(Z zY5LvF24dvP`3 zXuJ&uA~+NB;24e^_x1u462(QM@15Y-T2Rt-x0_81M5i)NMTL z?de@32yLU-u6xyf9g()bY}XB3Ti(z%roV4kT%bXW*(o&kX z?vj8rjkCdDo8ptUl>IfA8UomUZ^-HMO_PSDc<+eTy(qjGug*L5Ft;KMq}f?CjrdS_Qw;wu!h;;Kz3B zlT2|`anWyGy9JsXK+AUPMx+&b(>Af)T?IOAcRvIAQMe+Ux)uIe!twqhW!;2x;y#BT z58GzY_F%KYsa>B35AFH_(j2T}tC8VwR=#PIZkPR4J_WgJO}ABa7k9uK=gVh(U8iZ#Z(7key5&V|?+ekjHfw4?`%w z#kVs$^)WafCEw!vI`Hf-^05`J-J7-~EIOXH#LXkw=iUN7trxtnyuKAb9$GJ0mk8Io z^i9LVm@RSNN_ey$0?!<=6Fk9Nr|W#epwHySQhgDC?Wr zxdOYLvG1+u^C;IZSM*hIWqlrPZB;`jVqon1`m!CT* z-cEc7)Z*s%buR64>yYs#1m=zNdd}$T?wha#?F8P8=*5S+u|m(tGW7Gi`vz9=Q5-(< z#Ydt&z0kNv*%kT7*^MHDIQAVmK5bh& z(Z5(3PB>&dD0(c;l?7~^ZEU1$Z6p7R^O%>U%+TfqoN4p!#9Cw4-`|0^eCvGQ{Q%%= z69oJ4-vjRM`i!}grpsGl@|-2XVWhmOw+F$7KL5Uf;l-jk4x}U6#R!T&LN?l0sax** zpT(a5<2$qg=-R@z;?JITW>2^8*(dc3NISG)@sQd$uV-f0YCMJS?s893@m;y&m-jC_ zy0dSYYXklQI;$*~mS+Ri*-csQ0K?#YBleJzJmz3U)RefMebTOMZv+N%GET=Gz_ZQuS|!}F~0 zROQ7y=~(I+&~hyGoTSAb89Pinr?zW5(FXchX3<`!JHe@4+O}>0JarF4>gds~1X-qC z**PVtR9N*d7jYdS@BU4&5XBm^rc1&8)*}XJF#qpcUaWpgLVvK{~4F({6>!F{r zcxl%pOs1?oP6k^%Y?R@qV^4zG(+9 zkF>qA;~TaMTcYh6A+#GJmGMo<7&c!O`_V4U96}tI{N_C^sS2sVtw56c|g zT##!t*-oSRaC=&A3>G=*5N_MCt#EC5oDFElUMh0QL%p^g+opzRTj8n7i+R#^Y-`Z6E!j@e?Q=3m zL_W1$+p!%%&%F3Di}q4?ic`C^9UBck>fQ;dvlZ=FkY(DjJLUQP*S6rg7b(S0CPU4RYhVdI|H8j8}+v0Z(~5mV=#YXo#i@} zU4(X5q~71u5cRZJXNM9yOngj)IPY??!`+~7cl#{|T~;|BRD& z8Lj`k0z9ez>;stfpM7zS)tEBM^nG#}=9eh5-z3~uFs!4Ug%3loT-n*Zn4MqbLnGhY z<#$ftB13eEDL<>0b5-_DXkM;R! z2mJI1p*<3*`l&M_x+yQ6Z@eHsr4$nb7!Q3D@Y@o0PDPth{f3k;rM~{~9yk{>N2|(+ z;JcF`)A}y2C*#M%`Yz$>yHgDheV4eS5+3!N)^LsD9eDSI2YrJ))~}DnwUy`WgI`Y< zo#dfS)~{#Q@XQjPs=SzA_3Ig+rEkra^wl|;BO;&Lu6{ia^vsJd&uA}or#Q7s{dzw5 zsQWmi&Q|z!khLDao&uiKua5@I`t>wi`&A}>TpX@3^PSVRaA5fges2QnRFy8m7hvB4 zzdM19?T-P?q-9-;&R8(lE&j(s{DBqyeXg&a16;Ody0}4mV=nOG%jFsh+4Sq$isrq$d(4fBb0>-`Ev&K5D^Ljo5IT^N`g5MU%!^B#j z=kkU4Vs!BMGC27AsgP&=o!1NS<6-@saP{|2!$W^3ZehZs{*=SfF9^3j-Gyt*<7|LW zFA=%qp#@BOWi3>?NXoa z0Uvd*Lh5XVPX}4Xr*CWbYgKJueHwUDpI!u*_36d9_EG)?epy!M3V1D_UikhuhbNML zp2PVa<$U{K@N>G(9Kgzg#dWd+J|jEeQrT{Q##u|e`F-{F*U?#Bbu_K&baYczxM7Nq zrTX$obzkdTcDyWo(?$)!)fw5({c7g5K*|IvVNPW}oL;I&Y|p1F>$T zbiq(5oI1UMZasPfU7g;f&{GGf=&z%%v!jlFsP;H|y>ZdELir_nLcZ(Dv5vmQd<6L| z=A%rX&j(i(wNn4GbQ!y|bQyasU9dA#ZdtnZ=neAf^d|RVu=?lp*U{J6QAa;idmQ~B z_bt&A@?BSsb@VOfBgk(tA7%QO56%@=pdI2^vpn=+-`Cz)37i{eCPIc2-Pr8~j4#X5%$#xncASMD$Cvs2`z9iq zO?Ee`)C*@1$1i+OX|?2uWnjmTgzNb69GvTzQ+Zx1e37TwK_1d^%*As%UR;Z79p4f6 zdi?l2`dp?s$U{ALym&sYb37LaPZbyQpyS2!K+7@P8<;}R8%5?=oCiCAN%y`YF3n>_ z--xJB^Q_~{w}3y&kNKpG6sP&oapuL~rOj_eTG0u+wagt3W6+?-j6n|?;@($lpbKNs zI1%9vO>>>^6|c+T+?v_8;!VJ5#q#5h_Toao()F@$CQR1AZVX@NJJejq>W4XeR@~SSAV?%aO+3!#)Xee;^3MB!nMAB0_V2AeiGL?{u_k9ii>qj+x?G& zmTl;bOri5rNSXg#(dUhV=FPAslW^{-@)_Xn+uG}7zYuH&=S{%zATRCIevtdNknVGW z*E(LYZA7$y>dl_9*K#-;e#8(0(21!Ij+5XA%QZr|B!-06l%>R_LjJmS*-8 zfRqwca<<-G(21IZPxlu*`)gce$3U0>^yO3p0!T@2>2*3=Evm~^)TQ} zJ={#z&V3iWu}InPw-t8+=EnU}>>MY23hyX!O&M#Q?t@GCLEE8@0`6Zn^J5D7_UPd& z7q~U9sN`vE$rxN8On7hSO8pk{_d#3s1*~fY%*D_d zym4`L1LK6z!y0#iF7<&Q1D4in)<5pe`4iyuSUud74y$pzboGyQ_@|&}9sVhp^3R!= z>vVSS8u}c<$>qJ{xc}$ImRL_39fWH={{_yq&M414!WVg(j>r3nKVJt4TIYWSdi4#; zzL)&)jbDo#);a32{`6a1=XmZDo+>WZb?q~L16sDJ_e=V_oXioCk9N>Lt^0q#d9WAx zQG2O7#i?EDzkdWTb^i%z#a3ir%g*88M?sJAqo<`k=Xi+i`6h@9^B(kN$2TL3yFphs z=Kp05Cu9BxfXh`D8UHta54_uJv{KFdxc$$-r#|o?V6F564}q?gaXafi*Jb<#^gRBG z^rF1g^rT#Nf=@jRUi#EVxIYUaxBBch$DS5?otlIWT3N6D4t?5>@Lc`t5uDq4|0u4t zzL4%8_|aD$V~V4Si*-rs-s7NU`~FX)75Wj8o_0~c`Zsx~^GW=mwD>XV8jp$$z_k8q z-qdHF25+#NxpcBb2I_ghnR-54=C*bYKz~T*0RIB4l{>@vz!RX&*4gCt=p6Sc&?7|V zrmFmq&vDY`{~Pe!oZ&x!r*)pRtmuRnz^8rBh%68G(G5J<4-4n2cO=fOKaQ&5-&pvo_C(*% zZydIPmcG?4=|+-nBI%|`D{@9eITajaS^s0Xv83IKgviBA`wSO;M)8>7Usy^wT+`ElqYvk-Fa;Ss(uz6OW z%Hh;sRR8|qrTzm%59cj)_orN6;{BnX&BX>^uVX#bI!C-M52ZTL|F2&*tZ#w+dnbp(w)ESE;oh0P_zAq9z;9Z244xMkLwE^j-LkJwS?bSX%?da)9)MZ zf^++QL;AJ-zTvL8*6$lSocX@tZopxDFsXA@=OTO^%H>~R;k$p#r~SU+%gJNDZ%DlT zzM)>*?;Gw58vA`i(&+aM<-GKL!*Rgd?;Fyl^!tVf0;Xl9-#0u6FvRa;hU8wo5^$7Z zDW3uh`SiU00U`HvrCs*>h6m$1{l4L$fTiCzWStz!_YGNR7$?v=p?iMvy)$Q=o8$X^ zKt}2y#J=Ou%9vzuav&sOK^zuTXI+$R?kM4lJlHo;=x8s` zZNGF3uC;w7ZW?|Z*B#3g2YIN+_J1>Qo#UA)JXM@+^M`N7dG&PAGMy#q?3_$9hEm&U z2j$NNeXtiWwKs=TyIkJ|o%2NJ{G3kYSf?N0R*r|i(>R$_RZtMMNxYoK&x>NC^uXHlSQN_i&q;+p0 zXgO|LgtS7R)OYGv|0WN0F2N5Dd14ED!bt4LsPJ2x92* zPxK9)+xLQ&zIB$QYb1TWq-&8@8qt4yl3_SDv7NmMF^!|+jJlfJI-U=EXRsGRU;>tQB zPIbObbiUo`#9o3(Sr+8I1m{s_LU+_b(^=y+WtR>)> zp4F#vIQ19Re;s(K|AV53bNjmcQ?4)Z{!q^aVuP>Ou^wt!6K~5y>8=Ny$BQ|)|2Sk; zX9L2KX>rFxVi0lpvlbbRr?Ws{Mc5U54v=2 z&--HM_G}x{x&4~}(|H`93-em%_7~$^`(K^guLCTd+rI-ao!eiAb33=EU)#C;<+#?l zy~CNg{kwom=k@{%b6)1t&h4)xkD1#OZ|C-UZRhsafX2@4NuzUnIWNuaKLEU)+ta3W zZvP>`w5)V)|6#y#bNi0~p3dzB7V_zN<4p6AD3ZnW!~l6Z)^vyGHAA9Sc^Vf(;=2h!;$%I~F1K)=YGWP1=x%%S|ac=$ON4QpBAl(n}qi@~C z6h{>o>z>xtAA^?TuAd;S&?o*u{n{RohdO_bUoJKf>;YEmJAI6B^|5!jbAS3z8XNd2V1w1`q~tMvX)k^To75oD5T&&=ifTun}+E#qz zUeNJK`zwtVu+DNFz^{bv*GMH+@c&k489Vq5^r+t~$@+~`@7D#?Sdxn&)R2Z)!hO(d z>m;x5$B%xXb5M-^Oa0md8JoEe%QoR>?|Hc3w zZD|xwgN8>{f3%ahvd)N8ozIBQ|2mz`4y{AveGagwGx8FrW#jV_aVi&^AG^FXkmg{N z!Msv_S{_;_ZJinhIXRx;fb-D%xX#q$5j8v;1|AVFA%sed!k!&pMy{VCU%cz>wp(b9d5^-${^@irg6 z{$uYf$;S(gCER4z32XzIEwrHjgP6i#EwYa-Yzte{*h09MrY|3k@BVPzz~51hE#%F` z82$lT8)K-kCd3*Z0eu>4;C-{P2DTk(tl^)4*;oUwHP-NNoNNEBv4;Numc|;M1x#ZN zO_Yg^HPG*Etf39p8f$Pk6KmK2xHQ%vurLQ_K5eX_ojfMiK)j7L=(UYCYz!V7Yaorr z8sxkbYv=&IjWy7wG}f>=U|LohYuEy?T&!VBz|&ZRz(PJfZ;Un2E*opu3Nq4I!`6VM zu?E)Bp~Mw3;yQwt0jEc49YJj+uf`l|NJH#l2k6jon)V;sr*j_8 z^VVYz8rrCx=qN!c_OP?bXB;auItbUX(k?jH@daZK#O*44k*Dc+eBa{FZNIh$=(T+& z|L)|+xbWp7hxwkY<&E;}y^gg*9g0O*porg0!(t<0+q0^Lwz53dA0kAsosV-LZ&y4ZsfgxJF&kVW78 z^pIi?Dkhzru1{@3FR_QiV1ta+I(V-B_$r)RKRE)|>I|r8c zgVpP#cnFcx@Y5$eR9*z;ZW0B@# z4=-MwW$a-(^r+tqHTIyEv=Dnx(hz%?3B9&X@_H72^aGu1?+Jig|2?sW|0LnB+7o?AV-K@IOJAHP z>G6`DEa@poD{_oqS8$MJ{gvg$veN!h&zTo3_fDL%Y!>3j!}7DNnP=)*T%xo0r`-20 z5juAt!K?RC=ayZ-GtWzr>b=nWHwN%%OQTp08Xi^s(N5yZIwMYXo*_C{IGxQ7twZGP z1}y4~yu@kQ_`F1%%Iy)ktB~ekl)=1Gep(({CvBbT2VIV50B|09AJ>_Be0B}b>cHb; z54?Xl+2;tI+P?OT*>)PKI{VeFyq z{*>!Wyg$@)n%Lm$b*zV4=ZLrEp>!7l&f~?5JzNBt_49LW7XKIP8IswJs&*~1k3GBv zwx+R%a4$_?uCa&Lw3as)dpH%egTx-xnh<+f0Qxlc!24!n4{STq*ux^gZ0v#88hcoV zbM3!1_Ha62Y3yMoU>bWk6X!PeK)<)KhhAK3?7`to?4b|1H1;5{Fehg|ZS3JJ@|f5I z@iz9L*EaU>I?&kI18Fq&Am^pn!@0oQ*aK}!V-FVqre&qEhc^J0i#@y%@HF-yu#ivB z8)FZ&%f=qwgzGf+@Mgf$*aPe6P+||PGcQK$;jNI9j@>l&u+HhAo@AXvjmr&nyu@+Z z+n__oY1)5ipS}xlJRcaj&P~TXB;auItbUX(q**8ta~8t za^Z_S`1Y^RF)uu~{o1>6t?e^$SK!BS;gw8rkcWEqGJWCIxX$ssM|i3@+h_RL!&RVV z?BTtVzAq=!jKS1)+Cll(fw`gU%0#&g*hIk!LMCmk-Rbf*#W@4>k7i zPUxzOJ-jQ2li0&Gz_~eEU7d07a<@0$4>~_Dt0i>n9zF=Vq^)Uu2(VV>RUZc3P+|`s z0X>h8B9+)ft#Kg=P)7mZpl@CeS@g|+;Xa;~eSSM=X{EE%^{P!6TnvPD@#C;V#%u0e zee#olTVJ^W*Xj@C`2>FSuN#@-sN!P%)B5@;&~mKyX`~hU#7C%K+XV7Z=V$TD#UFw_ zz-pb>ys4jk0lbkeu^YY-aK<;b&8&T(52f*k&j2>0yCt!No4_}CT~F?s_mSdr;LF7z zJ`Z><2JuC}Q(qx%E57n2(D6w7F^xg6?sEOZmxbO-ez zeaMWW>#WIjtaY?9?q%Kj8g$$G$?LDyb>P#v$hWdQ*!L;$G@N~^_d9@FU%sP;|GUCpwI}+O#v;A}TKeTTCB0qJJ0<-d z(uy48;}slaS>I*3v8Vx55514;49}lxc>WxC zd@O?ZFDLsUp;P-G#5HaH3sTi5eVdC#JX|B^Zz6{}m=Bw0^{E_A{YCZv1H9D#i0EM~ zqVE2b>r1>p)bk^;!Po0p54Eg`x8R$e80uK>3bAE0KJVz)ERVy>`U}R(5LYT-a8wQVEd8ABYpyyjYsfW;}O5W zx%TB6kN72EX*}ZBfN4D9ew^EQ1byGeBYuZ#jYl|~iAOvDTpEuMSeU0XpEe%xNAj3> z1o1W=q1QGZ@mJ8;cm!!Q9wFzYc*Ng>&%N0k9Zn#(s3NeO|QCMHq5!snC!c5FOii5Y zID>Hv;@X5S@-!Wf?_>PA?caujUfXH%4`t*Y%W<-{Wa7#(*yulh_6DTufp& zz*Ao#Z7aUAJLq_%eOVfl*hA=Ej3-;OZ{NpnF1>`p}SL5^7B=F$twL zF^RpP+tyEB?~NaQLg%uWLzMcteaQn4)0o6qT=S>C5M?xr{U{3u`wPO?fluco`)7F? z#Q}k*;ds=S4+7l!@+)ij4;KEaJ<+c;Cb17_>6haqJy6m^Bs~;qMUL_D3J$WY@6yj$ zR@z_cIrF0Bek9IWHskT*Vfk6s%ro^&EYaEfQ|^l=3!S@<;MM!6bIwV?GtX0y>b=nW zHwN%%OQSdnG(4*Mqn*T+bw-@(oF+PtaXOnFT8GFx9k8e~@)D<^s+`@8ddCkLTC$92aC;J4UQ~Qs{HEli-sp^xy z&BY{6u90(!$e|AA!{%9iDu+{lQT?Zam--io9>yfEm3NgNGW8k0B{FpWvf!nut} z(D!XjVm7WdCgE@+hcRHwNJu!(`$=4r~P+Aj{I0Jfg%%=T__U%pJc%EP9S{`91 zOE>g$hMB=+4JAgAY(p7u=rMVZV~9ov;W~yoleU?46U6ljU*u^z9^dEqbKBnyfL_~e z^7oS;jNq^d0UW@kY?|coWhzW>IHkHFi-$ z3EyFVdm&`gm*+Q&;==4R;g(7>nclVT!U?0B7qaeN1iNJ1=g!rKF9zKD)?0C{{zIO( z;75O4#}r2u7wfCm@3(=LW4^Z|t&~CY2TzDfAHRQpS#z5YWYyQ-CqKsy7E#N#@?+ITA zKAi(ym*r^`9}GMV$D?{b47m0EkJRvgRQRj*L_gFR$ooJ`KmLHEACmNXNk4|PBFD4| z6&z$)UuU_oth7JYbLK_M{YIR#Y(9k_56jQ8W}d0%GbK8Ef69IRXNAt)NAT)>)H&`= zz%$REL#p>e@81}}qb-f%3!vdq)gSF7uBi+?FssD$fhcS@4`%|tj@%~WH z4Pt|@*RdXIog?0shtmBNa2_u-26CI(pW^3`Ih0oQf4M$muvVJ=#oe$yjfsSNYsLlJ z$~ujY4>7MA8~FtIvayjG12snSN${mH65d-IBVqfJ#z;O5n2nL}T4N-i$GMILG)D48 zz|t7W&46i)PlUqcIXWFU3f{54??$(55s-@*}{stTaY)7ht&<$&Ud~VCNPb2zoj^`X%DchUKeoK9q4%g*Hk^Sq$Pw&z1!*Kr?o z)x}I6$l)Yr@>k#nohNGDk@X!9gMRRNVPJ6UJRSjk(mpjF1+0~M-D98|O3dVO(DV2w z()Gklf|2W;&?zN+hyC%tAe+8?Rr7hpOjKGjJ#5{D6Gk_$WZiudcF8!=ovRQ38*uAe zPvctshdfW=M}PbeQyf)Xtgl+Xp8+k$i2p@ep-+5@`n3%q4|T#I9Mx}j1bcv0AJM$2 zU$%ia(j|7omjcfCQZ8mfpG#vV&jQwxFC;fX_JS#%1AlA2kqYqZLK?_hE@sjMJQp+B z0PxheNZX2U4FesIw69EKCc}lU9qD>vCjT#Wn?5%Jy4B~Ncg#es8%oSXc}&b?BiLf= zGp|SDM<3F;He)8NU+RM!lLuECGuZ^!{HgCm8O>r-%EG~VPxw0U>73~$S)N9*S>S0n z9@V=A;MVuItl{5E_^b9rKh&7XDA3Z6J0#s)(w9oQHPVV4(FyI(vW0ef`ct=k6nT^*-vHcqibQ=P{%!{Cr;T-x$E7 zEsbJV(D110k9HDQ)){fCb9d3Xhtt{Y&^kokJqg1v%|CHkHa;&^xvvnpdm&X_7DgG& zE9Ix

@*Msj;BT@$3gU5514;Og-MehUb95e*gw@b@*=L#=bf+wxGl ziGcHXF=Hl^A=Axl29HE*3;91?Ka$M2)CSi|v%i=E+tZjyxVL6ppfQtehL~54nQRNb zR$?YV`1 zCTs_6%w!*2Ys|#qOw43o;L@0hz(Op6`Lr>UapW;E6XI>mM6Yekcdft7R z-Pyk^{lI*AqLrgCF_IZ3?{SRL=pbCjShH}h;}XVIh?^~Zk*Dc+e4pgcZGSiy^xCeI ze-8OEo}4Fgn1AZA{qAwN&hZ>CJXM_SYkZ7kK4=*uIYH7Bb281?jdsvJ+CllJ;5^t1 znA)4esa>wmgwEH9&Qo(bk$o*Ymk-SAf*#X;4|P9|nb1`iBYAZWCoz(ffD7}&T%D2F z$jP9K^S>;ii-{}%UDEzEP6Mo!d0QvwT8W9!H@J>uA?SH5LTX|nS>sBvkQ{}?Ko&z5 zee*uVK)9ztQJjY}dkk7y>Ew64Y7+)n@z|rRi%VdKjPu;N`s6aet*W*mSemXNGtS-k5Iq13FM*99{h4Kk6;h5TIV%y>Sw*+jdY3K@Qr{o zzM*TbKF0U)P+lFlOZ1^M=CKm6A>A!mAJPrJ!RvZ**SwDutH76wahwTwF2>OZceM0BHjA5|8s879)Jh;*r#_Ms-pZY?S(Ja;i&V%)s@O9wRIm~%ko)%ym(tuMc^hW|~%U$rOtmBuh$3tIZ+xsqNW>4lQM8EHk1@$m`{vaIj2+*nrH zU+OvYqUF90=PaAI;m5=Bv#gnC>Ul?r&fcGLUwo<1x%&uSy^lIay##pX`JG7hUg-TB z19-HhQCtoh9##F(PU6ZsBTjW*Av)jfbT&J*4w3gNz@pB`OPrRC&r8Iq+^a?IdywW} zl)=1Gep(({CvBa2Kj?Bi*8wgohcyK9BON$mS#YsXS zK#+vs5H87u1d^MOn?Qn;~r z15Z;7!y2)6@P~e+m6|Mx5F^XZpH^(Tb z@6;Ft`44l9VkGj^7zI7c7{xf;%duR>C?+7!9HW?oJQ<^yihDIiLAI~PD5l|A#wZv} ziBU`k%p9W-dHUL&>Z!&kW)Y1Nqae5%qma*PjA8+B)EEWf$QXsVcZg9e0$hzzkWA(n z#Zu(SwvsW5UyT1jxJiI7h&Y6BES(JSn1luC$3>r^H!^aqYua0lqfiL|u(QhMq%nx@6 zK2(1yr<5`#?!>bznq7j%8K#aMJSOo6a49CSTU__J;;GEVNDh*ZpZ<>G>Lw~)i0&<9Xn-|?nn>>Yk)mHw(v$O}oJBdX0b(*gDbA}oEC-wCV zaB0qV7FQ>HlZ{aM(kBoNm3aZlRZK#c2T-Z=WM`Bw?d&q{4LpRX^$ZHWJx{T+32S zg7ih&)Lo*%lZ;8+!!spm3kDBQ?Qi7MAL%jWj{u!qlRR)qEcY?;Ra<`I zhW@Fbcb3PnD;blx16;Dp`{MdYT%U>Sb6lPHC^qf{125HfsokipJ%A_sLiS%#&}7wVcua!&&lT_N0#C}H4bLQV4qRn|2?!q zq8|Uvm_!lqOd=*BJ>&n6n1n1_k4Y4Ttmc@6-j~X_E@Kie(2mKen~X`k1b$L62`P|_ zNxTBSIVM4Ur^Y16f0$zu?~td)B_#EY$=W#M7Q4BEo=kR-}^i8iCXE2of!!ZeotHdO}Ksj>GCdUyuw$u8Y z?mzHt36sXOr^@wt97$m+F^ZB(-HkbfhZp6`IaF!f%XtUIH3(Kl&>1wIjE0Xn{9YaJ z$^l>cZK5wr^q4P}7ksEasGL&D7+Dd|u4pO=8fTa>*2^i z9bC;Z3wJ^~$1dDpiWr7Bc#|#H!lZ}JEcnpA3OC$L%R3kA!Q>I@F%4hHCFXrzbT4h# z5BI8V)x)#2AHu7PMD|#puK06?8Tu;q+aI_z=WBqg6TZo&sC?-|h=$4xKynq+(B%PC z+6dVte~{sIde>-A@Y)C3+CPvv5iKcPo8a<8SofKW6-;b zaWp}`s~AT!xT%p618E6aRDGuBR!C$+a_vkp5YjJcgKdZgPcjD57SEKV?HD{fwRXs- zKhk^39|1bK?(X1{#zPCyX*?K>EZ2g3)%H8Np$`@G&hi*`C}SY4flGGWUR*nhYnZrN zadqOO_yi{yc&WBd?M7`S$76X7f75CIO-H_-{rKh&0I#W4O4tQ$+IF9wv0+010 zXk|ajHLeYCs&fpkvM*%+6-B-wi-#5q9Qt!E-;k4Fj%6A!S!SFlGoF>{=_PfDXnP{h zDAS-Nm~0zf7Xv2wCJ4TXxVp+C9#mK2Z*C8%ld4Yj1)eLKe#oak*^eU>J?`&@<~yB+ zulcF}T=5c5g8*12c0=aLVc^oNXWmKVt=(hK)$jD&oH8YAhAXBi`5FeOIP2QYJtMC9qQ z1FEMQBl(tSlo$!Y)fkC%-G7`_`7|CelnPViRw;vNDA)Wbe#7M@2k9i&_zb%=- z%Aj)G?b~px>`BB*+|Vd7lb=woob$F0@l2GL`FIZN=N_Mmb~DP!s!JiDTq zD`=cy>e$2gb<74X#Z2aj>wH%{mAM?rLGqCt#D5{~b$O8|<#h#6H^?RCiC%OsZFn{ARohyFXK6o#w+e~uaV=f(=L|FSRqFRT;L;p%J+4mpCYz%2 zr4Jz*DsvN(tC)!{51`UU$Sx^g+T|A98+azUVM{tpu_aeA6S6sT%wz-dl4T2~PT+H> zG}=bcC(ky_40z0BGw5B#OnyhctC-1FnUH=-8{A7Yc#<)beR!rM zZO7o@sqIHT{gK{N{s_>?HPb)v^d#a0U_tUtiwmE}De=0q-HPxBQIp=_< z_K)LOe?j1}egv)TN4X|G4>;BNBCfJ8Wd9XKz9EZ;b{RPI=Ul!aC&3)cG+?sKtD?+n ztV~ZYsY67219?W71}(v4+wi&=Fv<6(;Cl;KS9!#P>Pr00?ICqi)v0^Hb4Bwv^65|Z z;|N8MAGo1;sMGM63H6^V-j4;Il>ZT)N#-ZG%JRfm$YLhX-0*oW_)r;C54FzHrd+|K zy~y%kf|kmECCZ_giM#%B)EA?FsGPrq3|y}bJ(M~}aJ4-o-h1TJ-+wb^qLsmz6B}zu zOgd&FJ>>uD{t?rPOUj_fOgtdFIcB2wtuil=F_RN$zmKV_jG3GSU2tF~>~KB2SH((6fx0T*AGa3&@zs737&?CfAWCVfd_J9w5c z69!XaCU*fd$4o??9!sElsxg!MM5Dw^2(HFV0>k*O*GOs6P+1Gb)bB8 z9Q4AoO2cHpJTi;mOEgq&DP`=;=7uJ_pmFA>jxjtok`=fV8_6NAIbHEo=4?`Sl7sl? z0lv{@hPBdJS#o%2neiMD)WWAuY?u1h$QWy^T>~x8ZyIZjiH#1m+U?P?oSx?s!=N}z zq}4akW{HXj^|i(b{(@h0So4S|t53Ma*2U^=3GLR?5*w!Q^|6G8TZ8Qpwl0CzNK0Hq z4=aWC?YcZc(ebgN)&NUP3{Y5IIA5PgFsz#p6B!X25!VR!@s=*u=7}*@GY$lT>$TcK z6HzULuTLWr0PpTI=SZRDLK_%rPxVjZksEpal+4UIJhgntGx*+O z9cEyuwBX>TkcI3sJNB{^5V!?#Eu>0DQ4d2VW}kjqs+`2vr)s;HitWOzG0}1De4@jw z?V3kNN7~!@dB;aYhKUQ2wxfS`_MaPlnb4RRv>*A9LMX2=QXcTjq@~C~c1!awT-bRY zMFAt7R$;jX>~jID`T-x)D8g>`GnA$KnlHdd^)>YTB@+FqzJ~IpuPLF>kjw-tX`+#K z?g~csO8Kg8NNcRWEhawB4|n#!=;%1^hdA8U88#mWZ@_idUXJ++*Rb+B ztZqVFtffhO9C3|~41;eAAxFcq!7bV*{08yij7b&Ywjylgc$gn_4GZzMgc)!bxID>RQ;{(2`pP^@8B>GeR4CPBdX_$QQL5)P2L_;#EKB%!9nkItAnV(T#>4O4*OMalKxHfae)8vClcIktf z2kE>NA2bZVr=ev}p{etox}Y)pp!&#DeULw% zxpC4Q33~`Rys1xpUvF|fY+}Ur^f|)V+(Sp;v<*N@b`4_YbClNl}PCh9!UW`WjtyF>k0 zUYNBr+%FN=jg2+tom4l#8%M|a5`s01Lk<3BFTjMaW3xmi_OkM7>A0fLLVP1Sceci& zCE)xyic`O`Zf_Z33ybb))L+o6Fr_P!39lbHkJfRoio>$ji+fmf&gb@ zinH(2Xl!W-xOr?LyGhR2(h7L=m(8nk z9{ou|@JtKi9 z|58ecNC%aEEa6V)e4=Bm?E)};Y{#+7ghO{y=ZpeR8&Y~O_H(H(%(`QvDZo+>el(Ul zMnm;Y5NH#P>oMRd+nscQp1UK_pKN#PE6SIBWmjluenPM~6OGirFtK5kDzhpXVh2DPkI5D<_LYowVx}VruhlgSI$qq!#zFo z_B7~G)0Ip0{%ujS)%KW56O6U1WENIO0lX&E*^OGKUCXJEfgM9Xj zxA7&5n46gM)p3I6HBRuL7%~?&1`^BEy+*snQY=x)RPx2CrG&V;d z8D%~R0_$-M7AftJ6kd5T|5Ev}btVrv>GcVUj*O>CM+gr~JLgLTM=6}~*yOT~EexTi z$mm3Li%p;7kPMPdhzU>BQLnpOJZr23@pf10CjlXNF%qTrLl9P|)>zDiSeL2~6!q2X z&>~5^ma);cF20sHi!Ni*_#puic8c+dn$-*T_f=$V<%1xV^6YPmvqnZn*sV6Z6|G0m zAi9@&(o@j~6*t5u+#1@A*MV>tUsg?>g+SqJ@o$0S+9T7vEsaNiz{UK@I8eE@oos@8H$7@ zE*qzsd?x+7rx%SmG(ONezAWIhj)xn5RN3vg7-)@(?qT(fh}D=o72I#2lnYc1pG1lq4 zl(BA~i+JE@(3r=%amZ80y77b~H6RdUQLJI_u3(L$>n0$2#Qcu~oEF=hn)xy(m{2mi z@HZ~u6mld*6yplv_{3WE;VKixREkaU5Mmw8c&6b@eWroM+XPpi3GQQp`(SOBLYs;* z$v5e3!sJckF%9|5Cm8@Xubzc`voE4{PmV8|4Ltfw$`_H{=7KMpBXH;9%6*X`i1a~v z4(#?{#0YgZs?V8+a_1wV`I*mY44cjXYbTqdMI8ADx?cz!()*vr~DDDPJA!VN<6DHOa}Oy-vnQx zp>nCdSO>6uJFDH$tPwQM{ERwDf3pg>Wb~gEX-G-~QODTw9mxg}-E4vhI zRJ+`ca(5s><+Ubz;ERFv@z)sMrwnoppkmygfHOiN^wnw2MTO-(B6OGh! zS1__0%2#de0G?GECIf8kpx{e1RIX}khuzQ|5j4*Hj5`)<&fZEYX&R9oARXJ%`{ z#`I^!wrHJ7`F@s&NbK;Ai;uPW(5k>8-zU~$59jlH!ljJ`c%`S~sP7ZWR*w-5l#FQ| z8_Uyp97i6rQ3k;G-BW@43|DD)N$LE**heF)uG`>ql=%XQY|#Ce_AracOBz8`{73|gmiki*l9@>QGl{K~Ldf-%`) zvnh~Ie`K>HuWGZY+|Z;JG|Ub_Yw$DLLfULf;F8^>5!X*#@if^i@sl>29{4(cw5*hu z$~MEKT+(JUh%z(EGGVhiFU4lRck$l(4P-Fe?0e*?Hv12rnazrK2r_mi@W=!uM0t_T z-$iPWiP4IUHgb)t&d^3zHf1?Onw*Un5BVxc@sh64$Vz~)RFQ4|Do9oM7ue;g9NpdxgX0=6^MIri+ z60H2lIx3guX=#OCF)R`1NQXQ&X7Eh5mdb=Y^LRn_lAQ4(Gid2AsqupRGrh0&61Z7# z<>LhxLQd4!6a4eP7$+DDb^VkTWo1Lk0U2$4{X}k7`6G>EWL2x`olvzZ>R!HTRlZ%9 z4IuoscCB)?YUyaTb=ums>(uqG)2daQ+8rxa3A2QiD_@rH=gfh!)p3TNb0SgDR*o~$ z@9Hr4Mfpz98D`MYyGf#v<5OOxtX3GG2=*zJ1q}K97>x^rr;ZEx@$6I%#*zYz))mZ< zLHg!`z@>4a5Ux}Pl`VaK5!|bFbKnafuEP``eqY2NmVQq1vsi$Z3w-*CE}?hYx$%r| zP=y_*(DLA!`i9$G3N0U=rC!tbCmBIgq8Prz_J|7MY`Dcyel+qYIpq7>I!D{tevZQ6 zBZ`IJP2o{g;VZt?@Sw7jV=tcp4_)JAFPYG4nXpHh{lR`oS8?WEiqC;hf5mY9#Y5M( z|GKWyy5{z24(?O84f8j1IN5#xTdxUb3W4qVkg zRu{W9R=#yL;)U2WW#rbb&GhLjl%d)YJ(oqIKWRr~50o$Mp}ayv^EHB1Fwsc8cLgIo zrhKZmjQ>@}vr5BcfUSHj_!143%iDv;48L(hQ&rG7^E2us=Ur8ROMOsHT&ug{srVEr zJIO)(YXP6?!rRP{*A-04C2hO5D6@_%6E>^!Qf&4o7i*<*pfS(8Dk4vvcU8i(uQfE% z63fS9<(c+q>E9PnzQ3(UbT=!HRT$4Y8k-k6=QoPLyID&ldr5A5??8A#q3{qQenQW9 z7ms)B*!tGP5*e@TuOnDvj1-uah+P6cmdHpxM<5t~IYKiLKK9l?p0kECKju*rFg^yW ziq7ACdIKhXmh>&uhRN|QKER{Dq{d>hMf(21SK!vgRgA?B!suhNnhRTO{GZ2A9L!oB zqx~R5J*0Gy-Mz19VTp~PeH56a;bpXx&mmX%3V-lYeFZ%?K%zg@S5Ut66^#@c@)ZON zFwsambp<1PqkPp@G{v(@!(>S3p*0hHiH6EmeMOKPn&yJWi66YGtdsN=fxsnS5iG7P zT=6vd3X)y=iq^QNXC^P~Tgppin_*He=_}fRj>>L}E0YzzLgiJM20#4y313l>J(FFe z(i(!s>?;~0PxTc|@Jx2aw7@SGpIF)XZW1xqb^*r}f$ep*L<&21fg^X;u=%(PcvJW$ zTb!`%mO=&x-DAAdc(ekH*|sB)&9mD9X0~mrb8>9EJ@Dx7L$;k6w%tME;_Ag(%TXA2 zw(~D++o1r}wmU)w3zH+MSZPa3EPW1QOaWvDESkT^2n7$-rs+8hiTtdpj_T&mi_DT7wO?Sn!O2cHp_}@+NB^oMMwdp7~G&Vuw%+IKkwCPCTl3hiM zYm6(NCYvVNrA^y$PtUx~40)++Gfc`QZ8{EgRCYYBOjg*m&Py5p<(b;*dkiaR%r+g4 zJk_Qn@GLvLPOLR3Ji4b%uAK?S*53h^&{*tEh_Tr8^*+Ix##*{WSppCRuynCHG2vrz zci@rFkYLU`jYll=0;+~#)n;G&_*=CrOOWNW{&{5f=aAmT>7Ij{VEpBk}{{CCgm~HWUFDY9g-K4YU#|qqW zxN=)@*3EzMey>eSW*L0{-FTES0f}sbbSPIb=sPso_1v_MJro~8qrE6}N=q2BU2`V0 z9Z&QL=b?H2Eb2q`_t4Df?s=4PH~(gP=doQi4qsN{`@D7(2od||C!?$>NVI>R#>h#y zmv%V~_rgBdHxQm;yq$r3sx#H!*bC%HJ6oq!anyScMk~vj?-Jh;XQ5K=Y-~o7GPu6? z@S=Rl|5t_o9(-UeX^ndiFS?ibD;4|#c@6vtc&Q|5S1a?ng{RSOYMcDe z-cx*r0!*|o)KkP~b}BMSU8a7`lHaK39;H5N{yuZQsOJHN=ZP;3KEB0```?uNA|;Ic zG0MGG(#Vfi?k_0!0pd4YNM`jn+)9cwHz( zk&8;cYB!guU07Q<>hDz@KY9J^TCcGe#-;vxdb!inzeyfpuS`!xKRqz#3!F!8@}DUA zvc0H%)q7PxUMTo#{wpQ_|FV9lHvLIiqYeM>%Iv6bxkzTw*HZ8EYMwcw)H^gT zI<~KN>dlkY_m5@x;gN)W0vOiN2$p|aCt&)v`^8%6tJTm%#N_xlWk0P54+*Jp^*GkzR8Dr_6-N>>49Hh4p=QGs zm2_B1f`u{pRvZuZ9~?KZJcCh8pt?b~_qUme3F&J8p~JiZ!^ej-Ez$DXA_v?|({~}s zPISHmixzv6zN@q+{gL6*tdK{->jNGV5@wC`MY&qzo#np#^yK+rTwbEDi6msYb$4hx z-#2sGK9Xl7lD|lS`Bo)rWMgV*!dvEa4+XVg>2TI+WBQoR$okcy{{;r z`p1HVwrym!b%_ht5{J~??vd|ST3$ZEJ0batkVeT<5znoE^!Lh%x)Cgl(`uPQt_2i~ zY}7#Jhl>2SS#MlecX#4bYJ0*L`E~IL;Ep>ZeogEgxrVg^!MY(yS@CY<@-N*-Sl@O} zt;3@@Paa#bZted^+;1C0lzMbm=xbp_3%rV*&Hm#&-={^!1`~e;Vb!5wJ!UDA&kBm21AJ*yeYo-n2oQ_!8V-`=QYL zuT!OoJ7CcF0zJN)#O_yXY!4VuipmME?+jTrrU?4vbI@9m(EqemAkN>`9IowUq`-w;6(wrAUNU{l)9+Ah*zZ4FqFZWCAS8d9OV4zmD8YXaIQn7h`|Jm0qH zbd_|f0rG=6jGZ&K{nF^Nr*3qyk!>k3A4`k{hX*BUc`yH2yV$E%c1ha;v{JW=?Ofo$ zafJPbgxLY(=f3?jIDB%^C*d!7UlQN$I;>vp&;Pc!=-%P+Bhr0>5gi=?H>6(TvCz>! z*OK`}L;2V4r}6Io=GrwWV*>KkGvZpL9dLYa_qp?>98CciZFV5d=sq`VHJr`bh01DT zz&1AyS>N^1q}n$|xva*x_~SuZ;gaDJxUo+nxNYCFPS812&sFa$ugzjB2w=U)G_9wy@c_DS5kAsmk?{ z>Npw+GU|svMc^aF*G-=t&O1AQGx_eUFm#rDXC~UTg?UI&S;KcW$35tw&p00|E|Hx$ zb*W#I8dIojaF=2XHus&ZrpeKw2-` zr+I%|A$$w*Z_43}_uQ{iR!KEvR3Wli1%{z3E-sN_RDIF4RrX@@tJC+>9H$wl2M*g! zzzxAfB3&lldT<;2%GO$^3?v75T7@5ny=;sP?_;}n; zx43MTf!mt%Q^tr)g}R*1yD`6&k41#jmgDH{S-Hur<~yFZ@S`#b9>U>uZ9O9J-Dwp1 zB`7WyEzg|$;9a4s@BJ21UW|Q&pMZ1ksDG#qTKuBHy;`oywN&Unjax&Jgb!S>yYT6k z_S$2ZUQ;}HKT=(rJwBFwn(P~Gwd0r({;Ao9_H!Flxi+>dYg1=jhGf^C+Nuv7QYFk6 zkIzL6)*8(qg{6B5XR zDzGq-xJQJE?3d=KybprdOaT(crpMe)79xW}9B z(8?7xCJSN;p8b#~|2#C;vLxe-2TmwrRNsPUe#lUjEdA{08H+?Yr)#2zo||5iOijF>|>!XLqbBsuxX`4@h?NeEwP0{ zaok>9p%TR%&zNmmxlTyqrggo2Yqv4lX$lHe-=EO)R1ySjpzhO)K)=bg{dA&%d~z*1 z1J9IX{R|$S+DzhwKk5_8AEC7UY?m}1+8mw6gVD&on}>Y0@8-LqUm)n60MVT8|nVw!!pNMu7@{BSKT7t>8;dL=!lJ920_jg=fRDFmRftC8(H zN};Vlo_Y>HJyYP1=5T9pFW2t!9R794!+cc66xjZxjmVSd@NdDrdJaF?xq1%&Ry8YOTIsAVRjdBh@!PULo@>xBHe;;tvbNC5Ip2IKh z9nRrD2)KF-))QOW;<~e>xQ$C;T)15WFTKX8uxM>zJPmr9)m=)QHS{@>|BQ_c0L{Z63J#J zf<~X}3LBe*ynaR|)AgT`hxnSz^~HgtU4mjPQPLJnaIr?^c0w?I7>6@C*^Dz>kPADT z40_)PdyFM6G+c798xk=2T;d2enNUx#zl>%#J3Y1 z_e;S1Z4xR==Wz0yWmQ^57NRG3Z51x$KwUj90%o#1k4wli>mIFDytFc&mjRQ1O|_3( ztB~Kn0(#P$a$wR6eV!#pdTkJ%eWMYVV7p(akJ4*{DNpZIo;O6zLog=_$0-u-FyxDk zR_u9%@hq?mo}%ik4C2S11sdyLUbY_&1B=wpVNI_Qu90Wpd*K789|4m+!}Tw{HVS#} z_0tJ1#Nu)it$Y3Cw2p8=4*5wQvM(tQoeMG*<|EE`xwQHAc!Sia@sa?A@eGH~Wel_&uUujEsd;G}E z$M2)~JUkoU<=Mu$k&hoWa3POl0L&y(9UfD{2WnEN46lQ|Cmnut*@+j)^pU)F_V04H zQudB^7MG!z+%6;5);u`fxA`XDtIlFz1iOP|#LsNY^2hiNu!(ne1iwbK{2fZAU%Mof zDY`em&UY_h@}2KLsdLUrQ?7aqiW7Ci8o8GdcPpMU=x&>khx|Pz@!Usr@_lgM_Gu4! zm3iC&{aixC5Dx(ECNA0RyV5Ox8n<<(9&dES6>-TtzL!RPHsa!Ampt+A&LPxkZ&mtL zw(dWo&J<%N`3#=G6)exCnfKMp{?QDyhhO$_Mw{ypxf)nSmA{lee-SJQF{r> zr*{;@^B7Wqr5lU1YCUalhoxLM=?dV~UIagZgbnRD2<-V!2i6T3{pCPIo>LM=U)hRq z+~u0dR=tDX&wzPhB?CM{m)kxi<5j# zJK!)rLD7egzYZ#LQRj2s0f+M`e>AYvp8k0b$ogIe4$Z?TT8@{c7y8%uerWOPGjtwT z0dExDr3)f`(j95b1A9+fk}~@-YLg+*pE0CEe6Pf^y{Q)L{3J38^AXRS6!(44`HN3{ zaF(9_(~L;%1}EHe-D%m`PqAoCY{-9t0(V5 z(^s1NttaazD_XzeIl40W$r`U=tT^M-Sp#7C-xsCta-gak(hyOTMN1~mJ zMD0Mon{)ietABpkUGXmUF~Mdb>3dgut$+XQYPMZY?+U44BmWgzq=6 zzO;KRdml}FCnD*3|3}*_-%L0b8mY)M1@K^7JiQ1#YQJFVJoT@1tgVQaU}__UG{$cd z#D<``vF&R^xiv=yt=L{4yvUCvf!_#c(rWetmCygO!uJ-L)$IYpsad^Ej))^6u3cJAKuN`U}s3Fn&z# z*tLCn5B%nJWv+v=Ea+Z5yt|yEa!zPYGwbq?ZV7hM((OIB04C)<2H^ z89z56uyk!r)9}2{q7%O~X7cO?uj5EUR|kB(uJnku8@^<^O0cs?hL0|gY3b4V8C$34 zx=QeqM9Xz`@IU@1GuH^%qVvU|tLy6H$%8x^>?;47d^FMRMbdS3VS!~Gs*iuUP3S7Y z4_WN3Ntm1F{|dyIw!OJbxQIkTIwP+pa4k6 zYgN6trfht1fH|^Q`{ewbE}fd4bb&`M6(bTBhoRwGhGruo%THbAtM>ubht62;LcgW+ zV)gsq*LADB6ZIi|kmUv2Y#0i|_$SJ5e68UbZobX|9MXALIF6!e*~aJl-otXwv5hf7 zqP;Ucu$STFsddi^-Z-_~p-leEA3By+u>GFW*B5X0QRU*(R0C(m?Yl$Q3|+TCZEt^D zXk9ap zOtV9wBhVB2q&>=&0hNKH+M9o8qa_=6S=P8mtN0%s`na=M6VeSjKeJ?m88b|M+?l2z zX524F-TeF2#;=Dy#@W?IoBF%L#i~^O4a1S8&Q=@w>(;A|vUp>PVzto}n|18k zWo^f_Lxt>oO(NT~Zlqn&kISuEwLRZI6`KPaGC1I{pCLQ;edXDw%e83e*f&mi<+WS> z%O|}*eRPn6KXAgUpk*xX_hXUYuR8h-C%lTm3<~u-u{CF9`j*UE4+TSa^n-i%y>b^9MfCx+&waPgw25ot^=ccL8 z_{L-@q+KgL`^u(=)e9dq^{r20D1K?eSP5%qXDoZ*ovYJo;Hhm)KBx{Hdl$C9O#Aca zpPK9%Ubl1-IOR2a!u+Z2SI1UTeUTy1(JU3tUf=l*_Drg9c`Aopt9d?qwt8*$wsNc& z3!w%^_hi3&5E&n3(@MObFfZ}a@Z64L16Cn*u_sVlkhe(hgufpAqSi&0A%6m|ZoS7P zgW@;Es5o^ZaiDovBG>ycEu>oGefw`(d$K;hjIrki66X4B3En88lX4i`gTZ$ic^#=< zllx8uH^7P?4spa?{7sAH18?tueUvw?-5Fz4lFl0gO{^t0QJ*AE==5^e-N6g0(0rEo z-$xR2#GhAY%~h&R>T;}23HA)hm?J(Zo2}jDd8Kypwj%gLqUCeMG4-O0_s?JC44We= zaJ}TouPGSww3<*{0NH z{8T@-Z!zMcf0_8Yf#q3$?81Jy3Sw_I{jSKya>uNp4Rf?7oh4XB1BT!=EvvQY+@o&~ z54TKZ{NI4LhVv_-vs_oTPP=+E^?duJ47IgsT|&Rb#{=TJ&aKPT=6?OE#Vw?B#48s1^&+eV^exVCo=ZH1~(Q=(jZ1mWEsY(9ytd5F|Oy@jS z_c%1=`iy_5eF*jrNne-5WsBeVKD22o(T@c81Yf2nG$E&-aM5=4kDL{@u1pDDHtpZd zj3nd^|F+Pi0&Om4Wn~krG{LxBP5SNFGr#-%j(VL^NZ$0a+orWkY@9p$AMXn@T8c-F zMiPAe=ER+=9Cq8A!Dv5i586-5W0HSOyMDRmwUT?7-z1rJzKTL^?)pXNPMv(>Xs|n48cj9J&E1VwBJ&B#@0Bw4`n>PkYj%y+FZnj=#TcI()+au zfN?$7;P+DEZIb5sla5bscom*5%BY;&Z>Mn_AiivC8W)R~%+M2h?=%tROhU>4c}SN@ zPDP&HiIrdOoEo^A_4}vj^HNB#86H<#H?8!gSMO5M-g45((twpk^5?$`$&4=DqL+no z`m|xOI|GhL7j|TK5{7xGwdJ#GrP{(Zt322Nkv@$AweHo+UVs11!7aKB-lj4RbZ8Sj zv%>C9Wcwr|hdY+>v`>Rg5&M43N@W;0dKtT)R9Ud@;-#Bx9YpYENZ9M)ht~~3iI}TI z(Lv+1%R4%@G2+h+qC08j2Tv%QIJ3-Rrn|G@Z#?)uf7-iljir+~wS58qO!{ zmYwro2m1X6|H!+AuCIT(xCd)Df*nRO{7|RqF@IjVdBSGU9wb^`zW+C)@6S)y@*IA>>&7qnTv5Vlc1ZDj8~;k7p*7J?22P!?l^k|# zVwDk#I?~!!;u{|{;B`M+$}=5cr?6-*i&ftwmuV-BHX)0(( zUe>(Vx?kzRj~`Y2uxxT+62^qY?^pjeGUUANPNw-EPg`CaT`272yS9OYKc2RNcBYHg zt#beB871u>=vO!GX_8sXmOD%8z{7qkSd7PgU(}DlA~!ZPJW;#Z=hN}4zqFMgpYH;n zPt!#YYCms|ihC5FwLSXH4Zb{nTVh^1E zGwby2x*ysCTKte)tc9j^cCB&x+L!iy9yE~67iC)+@Q}s`R`tL#FbqHLhM(kxxwi^v zq-Afm?fKR-HIAXr$?vZoH^+8+GEv!GQQ72Ykf*nUP{H~u* zjk;cs*CcQsNbti%%l+=?n}@D-8ZeQ+J0#dnBDrI!pYCo&)*$%XXR? zg*A`%_11o#zx#{oxo`0}fW%eG3Tc90ub%T$|6y0V#ucUhkvMvJ3}eV*px!5o+n*oPW_p7$)F;I2Hj?O*RWU#R zTJ-vw)2wX>_7KVFlRgv1S{^p4JB{}V!S50+?~@CQ#w;s)_Zt6RR*{j-$(J_lbMslh zmKmr`33d)i@00uUg0p9zIw2?eM9Op-aMn)Ro$Ysz_8;D?G1p(By@G@Z4?oVFq8$^n z+?jTGh~-;_o^Ysfv>J<-bhIZNr?@$_ucRj!rp~;G4mY!^}dR&V{;#_vS@QN>67nn(LI zZR(BR`|M)X&K>kPwI?y;tkZg*e=5kr)bzWNo$!)c@8{Z4J}rZ9X2?iw`W)?%3Y*16 zKi|DsZsmivZ`ZPZCRhf*jDBwQbli-cX$u|V{Xy_FM9cem)b*FG?M-k&wWB`{@yiH1^R|h!rubkAgD>>y7o{D4q|P&ZQa5r4Hgz+ z;{nkVocansAC8zlO&0aT)3v<2D)-(J)nGLFtR!&h$KbRHH*ZZY`)Wm9TEnQg?Bs8Q zWu5SL1LyeeUBlKyuJIVVZ%dCVGq%O3x_TSDgPgw6OyhoNytrKW^%5ZrM&Cv^LlW(N zRYj1jtEWfj)3qOm4rZ6JJ>K(o3 zbpOXSKVRNO=zk3Qs-Mxm0?Xx8Qg8;E4|^>1m%rRlw{!%E)Q1n*C@yss}E zyE@~m?2qGFT?iJBB>TE-qowO+R9d!@`jTL=NP1uQh>4jsc-$|)LMG}Tg7-jTb|SxT z#akcxz1oO4ZHTsUV*5-PZAsu_Y9erZoNU&iZk}7Lk0f8rZ#x4_?`_#=Sx#$HUGVR&vT7( zd7l}-2V4hRbz}Hf*ooTL3)c4@67(YbqnZw~CPkwx{2OLXnR#FFOViQTG_R9o%HJW7 zEhn4ZuKnow<)u`!N<8CsNccUGXguNjJQCaV-}}vqIF_99*XSh^J5&*RIUhWJLz1=-6+SCyLjXV1gJ2tw488oZPqr_=x|BO3 z*Gq!0CR(nSIUdz-|M2pJDoig4wirq3WuH{J3v0hG2`9ZI*e^)BUKT&(Jtyt8?j#en z6~UJvk>&b8b{XRhAuVbi%>rrx{h#m2b?Qf3)#TwT9aFK#y94zuhGM^r_s7yIk($kW zjz*R_aWgI%^5UI{<`>*Q?v!QT5H^4PK>dKPh}-ngUp>xb^`m_f$&lB9RuT7$H(QIw zR$p_Stqt9l&y21qb}eIug$YOdp!(3+TXAl(rscwZG3w5lp=|r{{Djem(GX76dq6f zuk~MK>x#1@RDBaX3_PXHF3%15ty`tnPhIfk;kuyMP;G)`^UdIY$^}5Tq+Lqc_>PY< zx%0L!b-xY1r&sGK_Vi(ce?%mFdr%~u9Fe==jk_EC=hL2BwM=7*C+2oq`aMO_P zE6;xbue`QDblk!e!v?Hy@@IIL9vh>Nw3#N2IQnVFEv=Z}*^0J3h$Q^k(GEp&cvrc9 ziq(%`$B_(wRx7mHnUk-};EW`ij}iPZ(Q9>ad-FNq^QqENuIP$Vb>> z#ObyJru(y!MH+a0e(L_8XbZM}0Bjc$`LiIjKqTJaV#l!+$IDiE;pbFuXIw?N zCJDJte}45gz4sv>;7UHD41-P)a>vVrnuijC$~g5;KtzHyOb{oskbKjoTK+L>r>FHi z*%9TDjmq+Tn646%VF?tD#jg~+C^>)nlLOd~sN%DT9yW+_IypPw6o&_R7THs%;Zvz| zq%X?vK<4zHQSP&sVH42LrgdhpZynuQR(36R_kD)1<#}-Dz~eTOu$LAsGuCL~**lTh z3&9>D8TL{oGCtp*tD52439=W0-z8dZF9S+9nHzQH>p42?GGNkPsy3*wuJ$(HZ%B5c zyMUzIOW4?esXwKit;%!-aK3)CmGAzlLV?MDb6X->9cFZ!mf~^$`}=n1K`4XsH*nZ$ z5y#1BIY!tX;NHEn zdDHu7r+#C5%A#WvX$h)#T-jT-ehW=k={4`S^59hqN%ULke6u@cocLK;Ru;khkc@u& zBumxQKh3x8Gid7&E$_D%7nUr3`t+NetlxA;+HgL0A%*ubMN@!{_=w|Mzfe4DRVTGl26`vJ*l zlPxh7qQ>~eH03f7{9B^sZ8GT9x_tilx}wFYO$Zi)q_@e^ zN|$OhA6#pk=y!tmK%y~H{I1i2 zCODFU{Z?-i8{5x<*ete}vc1a9)A(-b@E}Jx`6S{^^(U%E(V>>eKnvc_cFervP>xmI zm+?81#NkW{p(RCasRXTD$@Y6^AHB1Y=3&lw4zKyN%FBk0o*9rSEzQFuZXH{+Eio!O z-p&*YhXF#n<_C+k}>{_~t6HPbDncdlx?vbx&q(79OnX`axyNUShv zCnz?X2L!e7(fk?)wCw$7qsuf`AsL8w5;W}R1hi&xwtOM$bAN}jR66+z!F_3g%lCY_ z?|8M1A3?`UJfH8{#%04F-ehwlDQ8f4d>r;_+OTnxoQu`cP0N+tFS6#7B{`Xm3`1Rp zBh~4I3Uedgeze-!Q~P)OIq;#rZWF=2ROMQs)tUM8@d1VYxya`|vOFie5T+qok*U3k zA6<5}FU=1mu5XpH6~76gkO1wXXT0$Z6wcbxhCQ5rb;hWhp^k09whf4%VrU+Xs8$cW zZVU)wQj6EZT6n25(fRupYDKy=3iTR~1RLg4Ro_O9lwkIRk+-7Gwa?j_tp^m`<~^f< zXjd?;UAISk@it&H`qIP~h{D1Y^s+bF*Df*sP$LsALNkQxD8%*`y-r=t^+`8#JRMlN zznfER@BJ|);yuaVk(^VJgujaqUr@Soz8{~kz9iTjB*WjOJvjRC=`C-6<^GP~(}|Y* zyW-vrW)`bb>>l%XO8r?p{o9yx%l^zTHjw-s!Nwrz{_e*EU7k0om5t)zxh=?VE&FJ?&my|_@~bISHR?$ zyS_)|s|_kWtUx+Jd>13>I+zf;Cgrp3Hi{8SnU(?0x=}x zd>;1w%_$SRU7GB{`o?j-fhN>zZQp*RsWq}nE#kieypJG>IuDN9x-sJL%yF#F1UrRf z)VcQA=Fy*g`At?X8^MnfEwA&o&PCeSzTh*S$xX05NV3kyy#Bm3s##<)sx!fMBI$L` zoxaVGQA>WLcZ*V{y?}GKU9|N?-?tZSjkrw`Ey373R;e6Xy&k6)oLZYZ#u10nD{6M4 zXHb#6VQ;>7(uo9XsNOi68|_&8#os68H!|l@)*U3#K2HwsO`ScVT6Wex1bc*Jv`_Ib z?&W+je(_me4}#w#THZb{E|#f!|HyI!b_FomKEEvOygk7_t^~CY(OpE++o%1=uq6u% z@BRhtV|-`N&nT4sb!>UG31(PvR^z42@yX9;&00`(AK5TzcQYUa!S z&9f{$(m&NgFPbyB$LC&_)c~O=8(!euZ{K0{r@1rks+uI7j54T{Z8msBG}!V(alUqU zUnbrd2gT7Ln3{FWkef@+Kg*c3jIu4n$c!o4`Be=HcCR>%jiJuv;FwU+jN0`dw@ho9 zWkpl;qwHtu6D29k((!`ZPuv|lh%*VLZ1Qtqi-@X!6sX5NBNS{wo? z^z^&y!+q1PyqJlVLohGE3_Wc>G*}xsAQ5}P$=(Q_o@lwAMtm}Of3a>IEjr8tFsY|K zMx1`9ow$roIyv2Y(CT{HY)oRFUVEz0_$h5HJKocAxw+DbWx&pUwdeXJKAxbL;fXRh zp0;sdlka0&tn9;NFN6GQynOECPM6wk@jg)E57CC*;s5%nKCs_8pHEwldr05^k&GUI zHBsTG(r76@8S}{l0dD}_Emi(gcV|0TH`l2c0q+V}PZge`U*^ku^5&G^0*?aksVe{Q z%--FrT>Yn^pdSs|X)67R=cRuL?r`cWz-bJl-|1b!;l?M_{Cwu+-i6N=Nmc7Gs|Vq2 zQ2BW83(Sb~ zqBA;Zj>Eox*VXIFiEf(@j{LL-=f45;yOD%`?Va`H?#~Yci!j*;b_mJPuWhMz#=cJY z0^c)H`2^ofv|PXT94^1{%C=qob=Vfbq-?u})EOH1vT;W0JEGf!r0du06KQ<{%g??k z+U_`De{ouK+ih9ry=~~B!DU$+6W$3|ctKkBvAsV_``x+km@E%~^9)JUam)83roT@) zw5<-q3jkS%O2_jg&RXAx@69B-mq>`?h;z6pB-%QEzluv9A4RN|>Owegkc>9ko;}5& ziM=Pzr`V@4RW53hDjRs3t_2_PSAj#P(;eQplGX0#H-Y=Q-1?PJ;${##v zl5Fd(kdMWwm?CItuYVG7@#vOyY%WZFL2%L&CLCU9Vjo(@k}IOS)p@|jU5R7F&k0kT z^rS<>Gb88dmFU_r0UOEmom&D=p2W{$-of&8E`*IcZipEM6*Vih}h0q$G{PF3P zs4o^e^J^T9Q?6s0;J|=be1;nvi;ddNDrg~J7VZ2U(?7}2*PGUi^qmnO+z<`fAAGvn zCo_bO7*?AwGu5OATh%?pGYwHnQBs)Z~|BSx(<8%|3THgJt=&1K_3VjZrtP zdij)hvy57}7Hw(hHT8{uXWh9}$uX*L=B*a~+az1zKMx`ayIYtdaCxRp%Z4)k1Urso z*v;kc4UWWD9eSDj7J?rpTJBqV`Klai`M*lV4fQsd`O~woB6XKpZ#awnA=1)9KP7a`#E~+g1tZ5!MsoQrE77QeM@H4 zL&vqoX^90F-mBHW;yZI+NE~mRU)>FO;Zwh@cX-`)?A@rU^hGfJJ|?pfOoFkcl>PFE zy-sI@*D(={#blT^((S7r{loF`v4MPTF6B^YgK>m=43E9hGzOt2f17qLs?Lm&Ir@~U%KubuNREq8Sz6by19N|IQxinwn6u~%{>864ZmF`?6t$GsZWAh7k!%pM_OjG|-)w;VVbf9>>;N zYWwitkqv_WTiK7u;zuN$8h3-e;%8Bn(;Xt)S83~F7bcwmWY@yaru}tB{jeO|4kZrb zjo^(Hhpxh`?V4}L4h<$u?@QmAndhhG?=n*!8{YoO(MF4Cj+q%cmCw-0-mfwrn4W4o z{|y3y6-6?1vT)w@!;e*4@`&ps!3z*A*U3LGe^>pxA!`>fog`RxBsqpOsF>p3f&S-N z?1o@jkaV5gzQ660>sx0Hh3urW1kZsa7sr9;a!$^4wgJBh>F2Xp2oHnCDBh0umBesyJPCc7^idn7JwE!Ik> znOGtz+hMQX z{}iXg>H{YGpu)i$0j)!xu|4#}w=R<22Qz#87&m6)cG|Bf`T7IS<7Dkh|J|_5o_gu{ z_)D}pOo@TnW*)pYy2l^e$tSqMQMOmCy7MUPe#S{+k0;?bKw`1C5WX>sZ@RT>`_DJ{ zdh*s+cn{(@=c5&Xv3k+<`}7}{&mWF|`GD1<19)~r5^dY^Seb0ute2Xxwk23BlF_zf z`+BagwriV>w*kSUh?ci)i-bQq7GLkTN#`31m~300!oNIdb=$KR=?KwvMAF-~L6Nd| z=5JXYDB3mwupf|wOoem2+19kd&i+g$f(=6Qw?)wM0f)zFld|OdZr3dDnXFub4<@+1 z4qJ;adD(W{z#5Z3p>hc}63M7fm4AvB@cwix+L-#3;6so^eSCWRrRzGN(wA%=Nw7Xh zsE^U4JXX-B=<~k|6}i?{$4hjD7o=s(~vGLop{zKod$dE`jP_fipT zD#3WWmP;3yy7|RS{9GA=O+#{A?@qi~?fZ252Vwq1^(35`NJc%UOgLnH5U^km&94j{ z5$j#r6G*D!W~JNNcJ27J9i;jw_z?$?K>YbEkhD@eo-s(_QU<- zA2OK<_8St;!j8weDOPP`Crj{)m{WA-8P$p4tC4t|m91?@O#Smnhkm2{b@(R0jXKY( z)vRR0;wP}alVz^~T(oJ8Sp_aXjT_LF!L}iP55f5Re*aqu^$Xrx$G>SN*gk_U^o!u# zlMA(4!pbDr0VG}K@0YE=Rc~M8dqQS{A41~aBw=r^c0Bu??C+s*>c((iemMslHZz9mj~tecQwgN}F76<&~5E#mN!wfXM8XP%9W_M~?s=@9kRWhBwI z!{>&Kd3^TlXjX56-9|F%oqb-b-4%oV7jxYt_*J6iZ9j77w66v=&a#m8tveae7n^UXz8@y*+S3kwo1qt~pqKQLSIL=`bt7c-{8p@$J@hM#nQuuZXS-lA~UwjI6P= z(3(lu1396DUxX0NjIDCo^m%Ak4qD*l|#M1-L@y@Pm6m|({V#_N{%#=V|}%J#3zbcSFjksS4^ zclwIY%D;=9$MuSEP9Zt!RsDdz&kn3A{tMSD!a0Lv=vD7R=?9MLeG%u=HFo2#n1gK6 znY#=eyB$LzN4I_}Wv}X8yLOGicbR5>QolZqU&#(V7ku}w@n<|1BK6E2520@*k91rz zrb?c@Oy8()wjhZ%9~-~oz}-*cKVyE3V7ndKV?~*WL&L*nHs|_A@NGoP`()Q2w_9Im z`t^9W2L6yNq<3rkugZF1@9Zdw$$SW(>)*h|tquiDv)~)Ys>!9P8_AY1+Q@#hUqq)a z(FkKTjkWY!@5PW@=h1^lwa(tE zQro+)J0z*QIuD_{K}AYD8mYTe-mg9P^ZFj)LU+eN?g>aPboW|?9XtNXR`5rryM#B^6<(0`;JeK|Pjt-B z*Tsu~vm8mZR4(d)7-A+jJAtU4pGba@5^trT10#`Mz*P z?so}iHIk$5e!gPI=cV&qNXd1VaMmFiy4xf5K9Bs5D=l@_HL1G+);Mh4GKt6Zz?>AGGm{$YDSrSEdfLnv?bu$rYhPCN02^pE;x8j@)9ORZ{j zdHa6RV&-=VHqW6wo_!y@dfT#lZMg0ddkSE4(1Bak+D&zt|Vl zlj&I=;1om>?csT&P@Q~b3m?^Cg$c&%Tfo|K`-rf?=a~-vKlZ*eEQ%+1a|j}0!T{h>3)!o%K-8Dl`H_B;267@lIf?RG;d86QNR z9*~4SSmQvgGty0f^`qkBU1L7TUA&S9(AmxM8`OgzfBj7#nft}<{}+9=3USSccJo>I z$3EnAus&Gz^oN?u_d+N1LF~g8kXV}+kM#<>rG1=v?t)l*5qp?a2aZr1I3M0ECbF$i zmg$q1vJu;rvP;+hTV2oxk8e(DxApnl-GA{xuKtV<&TE&eeP#cfowRQyLis6>B=}&O zrpL{%FE{kz_>v^?`BE!&lrHs}InbYO(-hz-fW+FP^~uAV-*tbH$zg?vG3|}k_Y9gI zbaE~2gD5u}lBf^fnWr>Xt*T}u1{7X9cjs@H0(cZ?4r&pb#%A5@#r%A#&} z%=dTo0gc3;oX?0&=6@LvH;hf`nY7yN`fqxF^Tq$BcTTL|iK`}0`isP8f2||yoB79{ z=qt=IY>xhgeLEHt^NeP7<+IhtRkf|@@e#2JB6gB_+ULY)-JK9G5!rDl%k)+Mr}-@& zHtpK|ztstSbWd&h#diZ_@jVp_sMBxbd}Fp@e6^{fLe_$N?O)OFDgQTFGVw1AnOmrv zYzXW4am=lRHdR5A;O{04nL|2eITdq!4<+&WQXN`9uU~%BID)oQ1MvKS#M*e8_C1H@ z!~0ip*e}Ew8y-mSxVqoW0W)cTM>#1SaeuEld)n@RT6#am-;qZectrhOeaP9s-tQkJ zGX9P{O@K$}?>V_;gQdIbCjWPT=f(-iW3RR9%(7+Wt5yE9zyC$=>1W$#xjDah`cMAB z>0v3HP!b_#gtp`ewqT#oT~#9yH- z(^t>#Pg`*JkS4UqJ0pt2Z%B4 z4epVA>elYNDYOqF_7IY&4~BnBe)(Q+%Spxuk>?R4Q6CIi7ow6E+-@)9gUC}2N$7*) zm9o=RIvzaw?>@*~yOPJ+m^hcSNmOw8Kl|Wc^p*EJS#n_g@}7U}cTNZEgW(&e%v$H~ zSA{-^eRu{EYjcIt2PO~BXDZM>h}abodu)jI>wl~1diMQJ#LuBD(}!J0?~%s~lBf?B*$)!DAC(o&_#pE5Koa`ku+?5N9nLM^{^xn0 zwpU{@mt4~H?eQc zAhCUM$mg%d|2lW+6A^o=ew>qJZn&}&z zUGaMJHVO0oAKSzDs)5aH>XPZVL%+@Yf0rfm{@uBkcPW31hrMJm&Rqm;S_(;mzxQ=`4&s!^QB_$R>$~<^m{@3^bX+J3yHOHSnnC*bbJ2V$6@;sV{CZnuJpn) zH3QetzKn7QA&L6?rMR|Zr`>M9it%^kIRr`6-}?&=b$y(^2AC_{cZjiyN>>%S9MBw>-#%f_x>l};&ie8 z-evsk+WYR-Ug+=GXJwGszL*m7sB&F&%o*BFh;0_JlfGZenL3BIq%rXT;%iZs>8rlQ zVKPM-H#+^dI-$QWJmb1{&5^ITU%v&^>9;YtG21ZyzSe4nS)Ahh;s50CoGoZs;_tUd z=I6ADarlV-E?l31>nWLCQMk1`*<)-krtZ)%p?$TP9z`ibVr>`SnhNWj9X^V-9b%m! z34P2bb6u?J8!y&&h^wG1W4oOxeTP~G!~U_@Cy2Fz1lJq=-mX@%`D6vvC2-sqgEa?? zTf0*;A*I#SJJkwcI~?y(wlySN2g+NYW~{ZkreiOcHX=O2rHrs@-|+O~*TeEvtw0a{ z-;wA+8^DpB@zLzTA+TO7h4LH_GAwY^*#a?M&%`LdSU6OKetXRhRu-p+Cr71Fi?62k zO%8RsBcj(nJaTw6y_t;YIX>(zX|utS#iXg;{Oo8J?5`OWr7R^PYEeI_)lH)^?J5 z(rYQn)_@-H`dnK0uxNVuTTX5J(S9*eF3eaa!p|*!%cPI_d)uY$os7`;ML6bhd>&#< zseKeI3r}Y+qT4RXS41}Zc++ul#CV%K8Fat)fPOK7#QMwS%(;$FPwhQK`wL?5LLtvz z6l+UNJ~^kb?;s)G7iAfL`8Z@!lEtI^r<`6Lz=*%hcvmi$d&xQ*{RQQELE`-7?uge# zesA|Rf%+3%7jWKtcuUp899NB(Hl+It)#uN`qd+K+MN9DU`V z_3Suox}GD98RBQQz_lC6bV(F@v!2}T|EDxWK5MEeHn9|(!{xg%{YPafGVKZEXj z#DX9R$Ec$6t+L;))>Jd&JL0}5%lO>43xhhn929B8>1EqT*L3lL3is<4t=gkcp`0Tm z&gW{oYaE1_{R<+~efxRp)&*ZhhS8HJhSX zm_uw#Of(+X1Rcn%gl)QgaJ7+^0j&EbeZbSjTM4Q@!KL-X2YwTnF+{9PAs=rwXqZEF zw~v<`ze~_dzj}|u8KrLe$+NCfM;qEHp}riSumEb`#iIjOzfy1gXMT9;iG3w_$%_0T zeUELgrtf!1y}=i7*9>@>7qyQu5V`5t#$?x^7U z8q0&e#2wZt)nft1gYV4FAb!GPKGA9L0*{8`T`IiyEm_{R!Qu1)VzA~kk=e$ED$3ig zc&f{5{C@?%&QK0SIqXqN%=>7YZ)_fSJ%zeWzjMH|ndtRHHx5YI(frY$ZS6%lQ=IKAaCIP`{_Gbew@sW;4(K8k*ca>@S>7XE{tC2OPMqzjFob_qQdD%{vgyyOK_@L_Cvb8KYHBkuO(^^ zq8~<*aM)QJw~CI1H*}~M(WXAM*dd%nS|3{g?WZZZQ z&u!}w`{qDdk7nJRcy0P~d136!=&V(o{aqcXku7GNO0@|~7ajZP>xrssSap2-lH1Mj z+7Z1M!foN;VBT2aYh54mS|7Nb@*4h^gyUoZTsMm$v16L5Vt!q;Lg;lmE=3Fu0q2eD zx5lnpvd6F^#A`%$3Cc3#`kWoJPiL7sFk>ak&VfYkg+6$8cJBS9$2=U@5i5kmjqBYN zwpz5k@^diw3i>zV#fS^*9+?L3f>Jp_5f9gUuWuX3k10U)`pixA^ydOQDb45pFb^pnb3Wl$j43pKky?dmyp>_H@XzZe0=QZ#!GK%HQ^m^>^H>sp)AvH`j@VE)$fy4%wbyrBmL$QT)ceXPp2MO zf0WwbVC(uS9^=mt<%HI5Z;O5Ol#{ISm@^6)0Qp6yIt zC;GIh6o3AMb37NCJXx}+^R1OPaor!b=OmPQ4H9dMz_7OuY#l}%p>2WK9Y{i3=w%!o z>{@W-x={88$}+a7ut=L|G$|&J(|ZmuVhhu^PNN@nG^6*OLA|FTakemt>%Fnl$bJpH zI-Licxwq=`%4UY*+F(n@PAJP^ytY5vacqe5=<*u0CE>vxeqd34YFaY9mk1kKEYlc2 zboZfxEbI>v9(r#{s-lTc+MH(DkGT3{IruOL_ZA8ZnD^ny_-BrMS>E%8iPw{R)_!u< z`)v&Ii_V}oVdd7S0`aGi82eXl)L4+sm^XN)*|v2}Sav`73}pwB(wIwR|c z@pQE+%Oy;^u^#gJTwiQB|Nin;&5XNrUm(^RFkxSO_j-^#P;TdLp{xSRGJSDk?Xod- z?y}6dz-u4Jv2T#Lz9?0&Sij+3IqpA%zKr-UNOWHi%TOr`^G^>Q z;jAg-k%F@M#-omuKbEzBsG|vbaO^-m;yk?WyHOO<%U&(&2Yr9}XI@@6?pqS)w4-Fy zHL)_71B#d}@Zsd@*wjpByjEQ>^YEIP8|_HDD5x{tF181$6US%zEr^uG=St|O{CW@; znaLYB#f{mJE8Zp+=QT;*c)CyX+Rg2m7)pXJAuq2<9ZtR9S7^1SGsazf8De@owTyd2 z3-2CNd2N%L-Kt&D9O{g7C0~XfAEKh8nAX=;cAlUnP+lbTVeAjrltg@05!KR`g7In? z*_C4E#N%@Y_#EEIZ!6722d%6mtGz_+knW9LQ+pHJ@^s_FT&x16_O_`@aur@aFgBL zsCE@6=LV|?Y{dM9&jU)6(%5aQ+x`lDe$20WDBl-+VLMpHp0jkdY>JL)PGYVY)_itScjM@B-Bf_>KO0r;W7!=Fk_jB zb%K{z( z-EH9fTP;XJ+<%2#Q`=Sb?0rAtoglHghsH0gcfD;fg)SE{b;Ovl#=ExrEBVQmIwIFO z^V};1{@EaQGx9b2&9theC&;*qeuDds!bJ*~5N;L-y%Ei<^rRfeJMTZX@(j)kdSbCL zJg7@1_v6AiYuOB!Z_Cx|;oTv$tv=`;Ahgf?bV2iDN1SwM`ygfyN!b2OpPx_jrWN!T z%37f;(?&n*g|lwmw_wgfMOhO_WNy1s(Q0s%fk%JrGsFxbapP&bInhODhxi9W8%Ui@ z0e6UwjK}?F7DubfEmh2_MxRAl9){1Y=>Mq%I|OCP)~_$;%ey9wP8<~-(yAbMOPd;*wYVMJm#F|vm<+)}93 zT{AMh13McM`=Je(03Ua(Sv#ufizhO3%76C(ayC5-ACV)tIn{OcI^~hW7r@y+XfIdL z6#$9#;}vl+Ms42BN}zoJu~0}nA2{zCq4nSqyu(3cgHV?70m0N%*~6a(Zs9NiV8jPT ze=52=R__iR=ggGp4jAVHW6}%t-|o0Fj~#cgTrWsCU($7VO2(bwnLGTj+b>%@u~(O) zeV)-Jfft4n`Z+FaK4yDQdu_MU`4P-K+lX%RG=*w8t^I=@=DL|UrZ>V1d-veJHAYEQ zlQ(5N=z)4g?@P&+GMvdD@BJ~zGERVNEzoX}(6%H z@!wW#_`;4mh$o{gWApft(-trMdXstPir8>S#O7`4-rlQwt+Nhoj#xA#&gN4_by(d0 zKwEr9hw}~Mv5-jJZIh#MrZe4%YO4(d1!5^K9m zD9vrjNK8RfwGM47Ius~wSD^~A(jUivE5wP2=m-P z#}u?3%H=@fY}fOd?uy!%9lx^uiTDIaIHu$1;Ja*G)s(jyhS=A=VNbvQkm&g zdhhp@w7UA~CSw;b*ncz_600vrwlc57uR0&Pta)%2YY8OSoexIp9(_9(e7RT8?n$`? z&Vt_yiIsD_F|LVxNOd>5-6LRb%z;GOZAp38-U`f@NrSzf(JqMRir^PMp9?ywVtS9M zQ%}H#KmxnqHU(72t@|qKhu%JmHbcx661K7X^T|74BZgzxMv~98adE)*=mRrmv+n^R zW(|qd2jYmymFbpMbMG{z+o=QRf=57dWrv_m+P8KdSC~2v^&%DtiPWXNmX(_A+2tLu zE{J(S!n!nEZlbGyZJYy&u~vzbyUl;oHi^ULBgS0U z{fk=cEml4BmBV%dW}Ad(8dA3NZfLC8th0mmGi<{?NTdyx)S>fVoBA}K$Ykf-Z6XO$z6$}3-;@%+Rf7`vk6Rj7qb%4RLBl~bTW&&b2s(Pz`XkEWq zxkT?)z}Y$pu8cjmr;E`My3CW1{~Yx)*Y8Z}U)c&KGR%H5h`m6JY3JmhiWyFaYf?G6 zrfQr{?W!}!hxf|JrLm5XZv~0f+3(morQYFc%)KvSHi$7gLwBuxJtU!=xxUdC%^|UJ zpS5;umHRZ|G_4ac3nA9#dP^!U=?XIsAvORpM(3l8uh+O{UJl^o1_8#l-C;_RjP;%2 zH)x+R>j>pRLZ7*B>Fd2^tM|;ll&lhr2GX!29`8fnmBIt$7+fww=~>$bh~(JVP(Bjn zn4MEo6B7oA@LSO;thG7VMqD3gd{?hPNh`iLp(q_nic+JRDSm|CA0UyBEcg@^y${@<+Xv^#y3I+2U4#@d0_h0#A0MOz-MjDM?Y^fF}iR4pUNb6Kx9Rq`pI5 zSlH<3NO;uX0CNPMpMw1&M@dWn0E~R38m|F%?O*$)6(y(q6Z92#fO|zy%Gn2OtUyW8 zeI~{BpOh)ne~{l+3V5Y~#{NQEW8YFhQ_@nD41UAscTCt1Z7I37GN42jcu~HIBwf6I z`%M?f2>V;AZ4;o=Ai=S`bdrL8#Pqvd6RH_oKZu*!Hf804fcM|zn^MhL`Bos07gs8b zS)?ojpgCKBQ?_Jzu^yUeg!1tEP=~sKYg4I^u-wM%g8XeDQPOf$Tl|KP8u-4n67)*$FApz!TaPeLTYDE* zfnyl_w<(Wm3LYyjje6SyzeqLMT3T2PNJ~h7+m0yR=!CS9y5dUcAtTV_FHAO#wiA6; z3Cc#Fb%uJJqOVmMsvYDpW_nkzs08~P0{xdlsFu)9Yzyqv76#!p#GuixQQ+^Gp9Zqfs&p?RZ|=m(^%4$Rj;-Qmf@ z*kz=!hmNe0Jf$Yx0qRXYJtbQB&whaRQ2`xjpZ=i8?O*&0UOG*VjZC36v%eejF*VT5 zAA^zZh;`TD_Ty2zssW>NZ@)d2lyPrQEd_Y4>t$QaudXflQ&rU?nhSS|d950258PsZqO z@Ehk?BHtZoN%B2d`QI`AxXO&t+Q5rrbx-!Ymn5CMv6{4l%=J3J&($63M(U1bi({l- zWPZ?P%hV%f!l1_K;?38SSbs~St%BEb26x$_9}3`lL!V>%g1!>#TSImo&Y0S|t#oucIn z>xr0XnL><|Igl;Whb~iEk@yhG`azzsOrb1d#5PP@gc#B5&*}|;Z%KKm1KW!F#qEK8 zg86)(8U(Z?nZb~cAJUIA=ssm);?PE9hHx?rAKFin?x8Fnsec&ISm$u~Cgm}61*0dj z5j|0?9xMaf!EZCUrX*qHdLiYq8$4_w*~G~Whe*W$^JuwF|I{EN8odS{~IKy!OAws!x$R- z1<&gxG>#1re8GVkn90bwLY^J{{8LCn3{2s?DFBSjoA7LqF2@`4;GUXZgX|SU!8|NY z`9hvwViGMA2)~)q(X&&aS{Y#!H6Mwa>44?c^) zdO0U2rZaVtrbYm58y}w-3IC)gc_SguAsYU5#^lmIfIcaDAJv$fV+=1%`wYhX#3wK> z4g8tDXTpch&<2u+u|8=8tQ2*M4~JE}OdF*}LAm1b6~;dXRA1?2z{p4OPggS?f&caD zqp!Ub(1~%@2v9ue-{UOy?m;ZlYUG}R#9C=k7Mb&Ld_{URwn;43!h0(c^JJh5P$bsM z1R4_=H-$Q+sWFg`ADlBVpFc-sNtBVI#&I%IveyoIcLtA0{`M?akbay&_bC(mENw()87IT=VgE_ey@KT<^JFvvpwnTIh$w=MYCe(m#Z66XfIP z|1{PryqF7de#}K^e$2HJc=$0F@{pK| z{Vfu6l>yF=xv);+G1ofCBeo(j*Luj46mxBWeDRoz&Ewk1$O>aFtQSA#+6c6G%(WTv z#A7b>%l{H{q0jtJ#9Z4!k9f?5u@J^wJLod79DEM@$E#X2>T3Kdq_4r9pcmt=c+ea1 z?{Sy#v6)!tB?QEurwRX0QVwO3H~_~%q<3Tc#Nsav$RlykUX%ff#9#Y>#ze+ap$=(k zKjh;F=Ml{3&szs2%1BX%I2kEghLn2*^7(V+(MIHtvGTvyM>s!__-haFV!IEp--p@n zHw{aBTBs_$G59=|3&V7uH3Jx(Yn_`MrE$JrNTvQ;3l{{W@Fb4Z2Ke zMdCv!dkgY}WeQ~xBer4MBE*Q^3RdrJ_?DE1Iz;@MXAk0&e4n}xdL+p_fPDOrew;z~ zDHB&cY((Y}C&Tbz|4Gta#qyE*KL#4>Tn*o(JbwK3v=KcutR5@_+krMEK0vN1Nf^0a zNcqn}4$H5FZ{)}LtFitO^%r6Ppxha@4vb$5{gC(^;>bt(hw#0DeEj^M#$W4s@z-0> zDRHD04w6>?cSA5?K@vt|di<}4I*P|&Tp#jcFlia)nb*mGXcPMkfjR}elH)L8y{y=n z?KJR;$70x5{8$Xf3-MU&Jmm3XF{DW>b_ITu_=3b@*C0~Bo?~^zxlBkUhDi= z>@LtG7NapXQzhB!YKBQ!r2djdTCSPXecEXMv8iN&4) z&X2{gPU5lH3&tddv%7Gq{^&{+c*c9@!MFIL5t32e> zYcAM$(ivWDhoiS))iCo+mBAB}euLR(rV1brYyT6IqwuWjc=}mw3oZ{|=OxcUWvEtI zR`}>>H(xISGnOb&ts#%q&HQgUFgYBSM$*s5@%bv2&*J_`QTR}md4{eC`TTV$YC=9r zQHi4epDpYx<{7FQ{0{(q$TKY!HtJxWm%f4Z!aX66DIDHqPGxu}Xi$^^m+u=p4Bl4v zjUE}EG#u9!GqUZvQj`tH!^oLHJzOAegnLe@ z&%i5oy-Iz7JaM1Jb%-6nXTJhQJ{e*?j}9u*<=~3whu=U3{Seot;d)zK|3}v3ott@E z?!;M_EL<0g>z+qKV%J-)xU%Wyy_Cfd>2;HcB?vJ=b)JS&^}rq+mL|k(bXO17Hp^ei zVd+B5+Ayn&clAN$EEUw5A;iL;be%qJaNbDz|LC~ZfFE0_K)=!ReP1|x5wIzA-%3%k zK;sA3ZDW2DpkcI%508w--PtH3rYmqIZm`-^yoz28_&a&=4RI`K0 zR@B)MbhLnY4(rw&_=;fflX;NvLt=5XE#|iX+&(-d)i059oVnZSOj(B@x~=1sg*saO zMMrC)j_>LP1ENCZKS;EfC#;#rI<*0QzP;K4jUQsKc0lv(Mfrvl4k?`-f^BODI!N1; zfNu-5VJqD>Y03iD>k&PeuM9Zv+{6Q)Ztk2hFmJd>-LO6?z$dJa&_<*_s$%t#9&^{- zxMlA_lKO8L+s2L}I;E&iK$E&*S)IjnHrIXgs3AnvT9VFKR%aJ6of={~HN|ueY4Yam zsqR#7NjgWcI<>@fYK!UYCZ@C7`86tAMz#4QN#{sbXAd!*J;ikP64Tkg>_PaZj8&nM zbbeuV_7T&mBc@YVOy|;#pC6vNGwYfpo!?lU`eHf_#B>^p>2#TDw(`pE&}@9~?mOs^ z?#9k@_|kqF49N?=^xAo?*YRF3FEz@epyKKGV9WvXuCzow5!QEg%vShp`jZ~N@1}hR z9UO^!BQ;u9(R1^op&QrZevysxWKl`k8hbzP{}a!k%a!9hZ%8u#6HnE#{w8CZZ#@b; zt*9o-=icf^f-=~1!Bya`ok52<8`i%38gW7e`lOfE@;iB>ndwBn{!x> zQ0~{xI{(Ken?pEk31DmteCCYi(2{iTQ9YU`Cq3f(t6@}dc6OlXP~H_19zU^ocdaoS+B;QzkL6q$4^v`s^!mt$-bXX z)AiBg)hFODdIClDocG&hdE?WA?X(^#cz2B0C-^UV28rsK`LiVUyL3)It!EHVPsm^N zgo^5sP25qjCMBnu*0YJ%uCTx887iu0(J1BioA$Za(t1Ym>J#x7J&~e%cDB0VDtIBM zOY14*)hGHddSXQNjO&>xNLSi0m)0|jSD)d3(Gx4G=hFJKG3}3*o%&B>PdwU_6*|Y)29w|7}u{$KV@22lPyD0h3(NhVkKr9LJ=khR5rB91m&lp#X z*h9#dg>%V>T*9l8yvx@kI*{i?TOsbnF|ZXKw@6XRPEvwcDh4nUk~jx|Cch1r&xvn^r!I3Xtvp~k4|CdNW>08eiaXMEi;^|`OWkiVmfd> z%0Nh@Zo50=%-FTQiav{NG2|cNVKMGy?Nv@+*F`x6*!y1}5|LZpVzI@iPQLbtWkUW^ z9#(t7yU!R8lNX3Rhy0eXS2d9v^7P}zgRj28D^}9VfsmiY!;W@cy>{UO2l^etQ;`3X zhn<^|A;`Q$-EK{ao-HsD+FV?Y_Oj&mXoG(fW{Ar<#K`M$CfGQZ>P*KQ@Ycb zUet|x^ThN{5YwA4ruR}=q2bswvTIQ5Z91NQ~QZoT0B+}@C| zeKSA?=hHrra9)O&%;*o=4f6|t=G#qw>#E^){om!bg}UMM&iSAR(^5$2+n9$>T?;`D z-dRybstA7j4u|aq>4#AB3+HHu82KL)#s|wmmc$2)yfifjWXOIv#qirUHF0EYB>i?R zY!VS24hyf=r1|Ll+u?^oyUU4toP(A0R#zu5C7E=~Ss&(6imlCNze7 zQKky9y_%~}JNfNr&h|yQnL_OP))x~T+Zy$w>x6qz-VrI&w8_Tr<|)ZCOqqyZg~XOw zwB1)X#yj*IEvpW3i#{Y_eZ#VR<5vc}V$V-N91daN;wb$$cV6{x(B6Tn)MNRX4O7Mh~t=c6%t?G!(i>6(OO4;A&%o= zgczQeanStcsU7PP$FcN1B)#{O&g5a-+<2alzhPzZ>_IG~Z<}-r z+E+Vu3gS4AltJR_Uo(H_S+_Y~3J}LR$V?13>HnsKhmJE{KfK=A_nYm>n!&Mc`@&_;c+iw|Bx=f#ss!Ly)-Vs7k@7JMWzx z`I(0w0i5?t)$GitvR_lHPSLvYec!q0H@Gfw`>?r74%YU6$I2qM4=_&m>K)r>f6X)) z!@~~%&eOeQ<6~`^l=aLx!uZbbOQO5VYPXxFYT*G^7O{5V%RE`#+`c{ZBDTGz+lBA; z#){&>6|XAhMmO=NaePmAC?w(YukvSA>wb8y@5a~+aZ5;ayXtoaejfHg?h0ErVlj|- zW#_$~clx%h0kiiGj-$Bm0WE8}RLlR!#pq+KEMk$6c(NfqPU^;-eA1S-C*IHG5?R62 zVKwHuW@OJP#MX(*%1-f4U;XNFB;6)n9K-oY@CR$xHuF!kAwI~)G8~@0qOwPyVP{Vw z&WmR_e7uEf2M^=d@rd(c5e^T2H{!gQgu`t+&r~h( zf8vffFD~Km%i{{>Mi>nrk2o(j;qX?19qT`TYp#emFFxV$iC0=C)D6k*ia0Mu;qX@9 z)0-8oT1A)7i&HpUgR1Uu?y~7zl;_1N9Nz9qynea$=wtl2g3RNmU|zyG_ae+ec%6~w zhL?cG59-AH%Ruwz+}ECOx*7f46pK3f_kLG_?@uwsHNe@J;#bsx;(6PzEurH9{(aw# zzv#Fr)S*5+^~99=2ez~h{yksCUv%6S>KNc!baq7U#!^}b|32;RUv%6P>M%=??&iJd zZUU`?e~mr@auBcg46W`crZ!Q^-fIJdr@5)4WI1|O!B(Y<0-~V7a_6n(r4YI*oWI5^rNwx zJ>mR0NGvwzM755K-E>77%Ygh`A@*db`TC)WkL+j+ufYw%`0e%Hl?M|aXgJcC1I*zb zkXW5^gEviB?eZgq#&$sdej%o%=Mm(RTv$$Hcs_e?NUWU3YV~$TTI*giJA-17x zjDJ|cbLM~*F~#oIXpKuU5G6ic&~|KPYdRpb38+{Mu=I=dbOi( zC7kht@dBPh=nsk2S);OKY3P2ZW;BN9!<`Ufq1{r}-l(u<=4(8k%NP<1TbrsplR=V<67Vccrn&R1({Ln?dTC(fBeaZY6K&x_7>yH;5> zl0FNAhjC-_?c1}><&L+%iEY6+K@s8rB3t-Q!!IO%ttI*)594Ize1eB}T6T37%3}P& z#>Dh$p)1M1~roZe+xPg2@j*pEdV;|F%%91foz zr1AWVR+S^_#yO!75^KlKS`BZ;D)g?PZH0Ai1MT5rs&_9gIwek{9z%SAvLtR{;=pem zhV0+lb{*{XPs)vgvN_pm*V~(he>}2}7n_7bn>c)ticLaFsmmc2$9ApZ=xajbn_whA&^5VRqu?dlOPV z;_F1^U*^=EE>F7motDS=MH#LWuKbGnbUBU6z|m~^h^qk3D}VOmtB=3D{kn>l$M|L< zU%nu7BpK98IB^5ghCRvz(aNWA*TX>M+%c=VPUEsycjHNLz8_48su%PDoN zJmNP*W>h}%Qrm4D@JxlY`u(?)dp7=LZ$ z%PV)tk8if%+H6)H@$I7W2M*oL)n2aHo|eb>tP6}0T=@@{ok}%PUXEuOp#LDQ2{^C( zB|X=KjX681p2jhLE8xqwovfu;w06N$Rvz&}QTe$S%hznXcutR&$N26CUw+n|=k0WK zuMJ@35&tDBFEIM~$bb03L9{$xw|gOxdCijg8q+0s!iHfpnEE4*_h{T2rCq!9-{d_$ zVF&s#)~hvu=g+W@Kr2c{N!JYa;Dq@S*o7GVOhSe^ZRZ0+409PRv_9c=w<>AK4TpXB;``#3r| z`d|Y{8-2n89DRIUy#(;euA3K7Mod{KFNV=F@b)V|-&Nq{L+4{#eZm|ZJskZU9m24Q zVIGbGXFnIVjSL>(XKUx-=xpmP!olKi(z)4 z*WM0x(s3uc>9*5t2&4T$#8ymxUlZ^XCSR7Z6_fAe`hdr?s!RHA5yUYRZ}GhU+BGu%;6gHfn!BrtxSr2VzNtA zGVBTii@{*OtQgqChu+5~(Bw}XFjzpJy0QO*wzmjLG6{)_PD)G-A%}c~_&T_Uu!l&6 z7#f7QCPc=kMMc9;Jp(;Ma|07YJ%f<6ks;}^2??os!(rT&Q?`J5^@k+y4rg<+V-KwC z^bSM&*gv_WF4-809y~(wu$!L zMtJPK{NbtwTiAhpo3Qq^gf@gpDiYu92sDuhfkk0HkyEx}Z7?}{aK-N!}=7o-@fy;qp9;{8&H8 z=kk2*N!~9m&&M{9ErXIG{RU$RnJ1|~X^TuFxR;ZY zhnFogUdsN?6DSB^eeib-D+c@dHdTU{j2^#i;2L3T&0J9|L9QWNz&C4##%gzl06M>^2c)g!v7&r)odma$QVay}!qd)QDyliJH za0qmDfXE5`P4kZi1YdSfqiTtPnscc14d5%Pi6fs2XV5mmy;hv7VLQ0`cLwFlY{@D&BUG9Ir0N20<}f~icqrQppGHXqIgB@h#DOA9gAgmBpL zZ*(;6i!#D|SdJ4rGHoQIiD| zY-A%DB);UuwyV$*pDi=%7x}THJs!pcHb{_wiNuuwu_R^#)ifQArY|&Aufq2sTqAuLNGqTea8vi(9 zd1&915ZL!WAvHV%)+gdwb0PE_4La;kQACdEZDAlYS9qA@E*A zbeLhkkiLE4w!y&2F~o^pkB-ON{dK1G61qtQll&)`}8pcb+F=TWNbRZeVzPJp2vP&`pmrHR=m+ zdX$MlUqkxa)X;?aHe>Sp8ky3xnV|vu&G7XzqG?kD6B;))LfTclUqdu>;lwNY{6C98 zVBiQ5&fj{OBDVPN-5tx`ar{$Rf4x(QOa?ptL#6+!XypI48i-|Nm8B_)@`vjv08%OV zQXqWuZ>ds-LzhQs=)%+^FgAjFcU!pm6ga^Bv#o&S!gm-A;}87h;1%c_7RJDRq1PSg z=MeA>@r;uzghkHm?B}4N>*)vDy&(Dc*@FW(`vR2-zU1r!e;i%F6CGW^SN*&-bTOWU z{cx3=Py?ZsK9GDN`9TVT^sfG_K}Q*r(cR+bEiPPNW@NXlb4i}h`-d6KvLtBdzROjw zO1J(Tc%-&_zDbQw^{u9Yl!;I87PL|-dE0YaN;5*QIyGy-mp4X*rU&=>Qem!EwTV)F zGE0hU2HWh@vd**d(mfEF_AO`Bk@tB619BI%QT~vm*|bJ(*PwSXTX(+R`t+LgIO!e7 zuXS*IzfyI2Rquw8jqzoiOi0O8zS!*etsU1=NsaOK*yEO^T_0@G?a}pL8+VmlYK+g|ZOJ`1j_8XzqouadR9y&)7TJU^*K(+n`M?zm) zqTuUixP}mEMFld#bG)U(oalp`R+f zZR>KhcsijQ`drUlCH>+&p-=SgcJukx=#M-Zbko#Mud*M*>~HZPxo#T`iJV9Q<*&~_Y=CWi=X`0w?EAYeRx*zkoH^L z4-$I6%@wD^qc*o+*ll9UpcEgw(rvjp@5fr1`+lx&-@o!nj9Kruz2)8QTb-D4)pB=3 z+_n#c6y95|>*7S{d!rl%y;CR#N*)&py)iRR_gk6TVM6=nZ0`S~=F4ZNQ6 z`PUmlpZD!*{i$V-*Myc&tGs#8vc~1bm9qQ$TFh7ep|(caVpcP^NgxQ^tP9n8@y-0`*>$+vX;sWw~CV$NcfvlrAaI@FSU z!rwH-dH$e=kW(Z-L1)T{m9_O@hl`WTYR-A~Ao)Yo<#JPJJo{)C^e(fjs&va)ef!Ox zr96AH=4aO~3)?OKN$?A|>?iC~^s9g2G)g6^Xm`KBy8X3go1gt4{39l8>#KHbVIbi* zne*JEyVum9_!%Lab36hKboMOivS4oC9|Ygk#(rM8Rn>Sxe;66*dvtg6tAt*al)7)P z@g6-w-_UHXbfL+FPe}?(a%LO$-*Y4NV8yA8_jys9q5Uh9<8Zz919e@xZgJ#^a336)!S zU%fTn@9qixpW~g>CvRTtyzAqrsa7E+YU2w-_iFr*nXDey?b6aZhZSAVnWZMSx4i}q zzG=X&3BGn+#;I6^e(h&9@Mdl^DX)x4_TP;@uV4KpS=Qfh>!8J_XLmH&v?ye3uT>4Nndy#xz1JjQRoRTNxz{ z1-tiK#!AhPnH(kaO(Sl|?0e6zZQhjNy;?vzyul@`ipb^!xSKZ*p5yeSQq&CSUd~9hm@VwKa znSLkVr5v0UGv(s7hGWw^|M=uI?qslR$*DB;(`{z&^X^9Ms}waP`XXiPMC?oK5izr) z!`nL@c6Mk;T6VO;t4glYQ1`X9dcopnBid;1m&w{KKW@-EBdt;5a$WXLs;Kc-J3{2l zRvuWJ-`UNN$VJ9#w)Q-+(@tFOM_pUfb(%+(5joQbv!5-{$(u&xj-JdKe?eh}x47I9 zCHoq~rGZvNPNk)z-u{>t4n*$$fVlj={+}$xAgU$R9x=oQkmBR8?zyh4 zvRda2bBWxw6Qy^?&HbQDfYJ|9)()jG7S#NFaiQK+XbNXC=J^TfctCyqP z5}wXn{F|Ji^|5jDHjW~48JkY&h9!;YL*#yxI(2H_Mon2mY$W8vfe$TlMcAPzx50sY9A0oTQL;XXwk9u7Cw?L!hyCVj5S`u{H@oe)QD~X#y^Py^CCQfQE2z{vCN!#yV9U=&wJ%91ru)rPWgbsSY zY3sakrNanqwZx)}>68qvJM6GG9+TaMyt|q=lrW@by z?6YjcME8=(UEBEtRG5%)o{Z~nH+|Wp`zxZUzw_rQS?w<_C;IYd?Q}VL>re=x*Vvcz ztqJ>O5bD(BI42uhG`` zc}eIWRaYq8p3Ykc?H2HG=x*7Qg@jHEXlCeU-7AOC@n5v0G7Ed;5_)xe)t`R;d4`1k zHQ@NN(_!)Z2>m9#K<(qIv)+W>7bK_MVOc8=LQ9oQEIog2O%Ae zx+}2jjs=8H7=9{!Ptm)D+F!oe#0GjBoY3m{C~AvK$%Oq&=2y*GrILSDsl|GMg8E0< zL0-)kByG&o^D;KMTlA&&<+0KgWs|)KU$|gFebw^=V;bW-e1uw-qx5aS-bC$|7mnNs zCGksmqK(JGH}SppRL)vzbxhaPyyVO2V|Uyxd7d5Esq5fzXCE!C4J;fOoj2>GQS-+( zWea;Zt$cYb^~os%0$n6 ze$>I~WQ#|~4ke!}+nh6`)6$$)!Oor&^oyQlXVn)rzj|SM&8z&x_YH&ULVoNRQLfmq zO6_w~E4_nb3J#g(9}IsswB^ZPu;$1qeZ2psU6;3JeN1@uDx~$9=hv*iy9GsbP92-x zZ>W6BTQP^JUYEDaA8uQ#bTi=N;=Q3eW5RM<=_p@0r&nSnCp%}_*1dy{$>~-&-B&d} zaYW~~`MJqmdWSw=yY9uZmNOsU>Nl^c>Je&FTD;DKxxe-w6nI=c@y6uIm4bkh9RUwI zpV)g_GpdQ+jf>8uJ+~#_Q`;XD)YWC-;MIN$_VkGCHYKlrzb{QI+dQbsN)){6*RjO! zz~I)^Gn{-|#wXOz< z$0I~()##b!7xk;<^ojj7w|qZ+q1)o1|8M{QxAxx}GofG&HFnkH&R+A==grX~*U5wD zb3ZR%F+i&O%X@pD9&5UHRfgk^fuk+H?OPXjM5|xHW$)tUaqEoz$D2+aA-&t~aI3GE znzrnqd0=|q8pPieGn7-&)W$SasttTa`6d1)({Y% z+FK@$jv1-vY`W}CV$9Xs8aa-Jd*oO9`K6V4w4Uhmsm}^)kGSPQH?ppLRI$D{$LGnk zc9&lbTRVfQJ(GLvlkT`Eo1D+j7j8PO^xjIox$LCS3C)jTUE4vbWBUtb(eqMACcT9 zd6CIk_uME?_mjz0g{BI77S9fP)-Am#Q@5jC$nwZZUI*v9&n;emVaxTWZnp}mS6mZ# zYh4>1uw!^$MzbV~vzrFBIX$b*oAE2{oSu2CTJY91&41O4ix2w+?Q-kuH??i_xt#%{ zObj+0d9$|Ety6HYZkO}3 zyryE1PrKnZZL}==j-S(YZ0E0WZ7=VWeYvrI*{}^A&qsDS@MzWWjpf^ooXRs+pV@7C zVyNlbC4~QFkDpPo*GsC2KHKQ3z{PVODu%fgr`vkCXAQB<4V>)W%+2-k{mb(Y8uT#s zjNjyyI5=>APF-q5b3M~;hU33`%YAx9s+l@-myH~jg)_L{<0%yVr^cM`lXHApMQ2jKnmgC^CGLZJRrX>!OM3R$hXUCK8oNv zn*z;@6|>I}T&;Y4!-6J`a|k}DWI(mM@%0k~x2afis?Whr^#qSCY;awWobFEW5ZAa* zlb7yWMDSMw(v{*C4mKyan!e2D1S6vef`43HQ8~HK%UFWfG#uIE@~&$!!IkRo8(g_I zz=z<1&7oKOH<9Z|aPKbL9=@)Qx=ip5-cKKClvVy9c*4Td2it86zDIDWewWs59V0uR z;KO1ZN-bB(3?=wt#UUzlFSN=b_>vy224)=!FebQT{*Mte=X#_OeAAdw6GpgQu_t(B zc7lHL@jJ3kDX;Iuzgqp;&S}ZrEwfKW6aI}Or;WPPBkIVG!t3oPjPN|`vpQ5;MXpqe z;L~4sn^r2fA(YU|v=w_FpWif=(ChEzoKb!4?@wqIokyK^eVKla(7|oZay<^1?;`Y& z6PG&d+FCxA&@pYE&68JKbdk{6i*M&0?e)-s&^GOd7`!=i>?EOUW=+&jvX1Xc=%6R$%4^3vV?wI>TVRIVK#%%K6xEG$E#<5BXq`%Jqaes z7Sjo>7qsoQkD?`4p9x#b%&+}=;6w10BRgH~+@u}H-*0G#dauNTu#Tq*z5k_`hWEF-Tzy>9(;BiOf-?!e^k!<|&z{k(2>rd!@o}$K=b8}O zd!4jiXVpNCzn6wW=*2gEIeX1NT{R}K+BTB#C+*wwqP)tfHKA*+O__J)=gez_?k5#e z)8W(Yy@b}1%8PK@(6yY23%Pr4%BcXjAGp;JD z-|941dzJmiTaGs#{e0kbp{CL;wx8CjUCm3jAGNZdr`2TSWvA6=RTUZvZjaIouNF*o z&>{4l+?Sertd#1^e7`4Tv^Kc+LuuoKHqy_P$n%%-LH%EC@vA#P=v#HIUmw|ZP!Ms} zxN6hFfNH_O%8`eYk50N&)UYeZ)K+Pa=cQTwQugWk=XCH*i&j=`lYDD>s6*KfBHvQo zZPTH|%g+e?)2Y(yMW2HWgkHCybF(drw+@(Ix@yzl%`<(!JzTr`NPj)Kqyy`v07xUj5d% ztHqODJv~yU^_pUFs(yKd|JJW9*9E8;PEx+3x_QO589j_mCtc6Jl-_>Y8vPp%R#{6v z_HU>1>|D>64=YAp8~m#G$P-dW?A)~`Nwu)|x9Vo?lrlT{h|W7XLEw&>wddXLbi7#^ zzPqoA!0)c{)0b9f?)N_z;N-9L+B9y9W?!q&8x?WJ`8T{gwI>+8+kRW2*V?tVmk&D) z?zGclb7!0Uz_^q#dz{@ywsSgmoak@%V)V!jr~5aM@&`1tTGyqU*=vGp4q0N_q&QKU z;Kwr$CC#~U%#h#}Q|mqzZYnJ&c){+M6Wckth7f$}+XUU}#{OPKte)T>ZpAme6 z^VJ65!B2Y;{Ee+$s~ZNsMg-qhR6Vuh=pmN~UNmObM=yJ=X#`(XyFEUus{X6 ztJ_n{djx-~85OO1KGl-oP3IOa9a}s}hu~|wXTO_&Wn(eH5AXije5$+CD}o26FDcT! z{dNMu6PoIm$}D)lhu{z23^2M;7P5ulFBFsxKm27ILvZ`S^9Qc%cl`mu=Xls^XS^Cc zoZ!ayH`aYz<@uTP=aS2X4gS>jGuIRBe98>KmfkP8o)u$2==c69 z>$Sam4I%WZ+^Up-8*{!8`s>e!QL!I2Z3w*v9(I!5qN9I;(9WM^_EgBNi6ykw zfq-omY6gc0-E8;Bi7Un^ZYH#+bC1B3Jx$8U^|tux;?euY&h9|)b{m~`9c}yeHle3f zB(Jrx*|UMr`(FFpJCjjAmC!rhZwRxYUd9vp`tB$4NlWEC2z~1Ey}X!FaaRaE$uQZm zz^KJ(Ld(7C*73fS!fZkx+2NtFHFEC{LN8F%e=r5x?q{F#-bJ1qX{klEAfHO zQz=bCAFowAw4nKQ1wu!}F5Mm;+f<*>RWHpBT+2`|CUmCj|6=bxpqse1y@3l%F) zC;|*P5Ma7yXmO|xRS=4(G2PUN?odTHHKLmqOgG(h(-k0KN{A={hbka|LW^mM>EAx_ zob%2(Z?Tfx^6q=zO4j1~o}d4F@0ro4%l4t89;te7yuJ}pX?LM#MVI1rb3=hsNOb{j%QDBTHlu@cQ7j zabcGA6FcGc(v^?L_qp#XjMq88yxTbMm2xle`gM&}pA5cyISXDFnYVXp_psix@Ooo) z&(%pA`xL=z+l-_aTjZ#6cwO$Q>qwv2Ml12!^3$EcU5X6qfY;wOuxD<(XxKKqKGlBj z*ps_f-^J^1N1vX0X>qYuczvU>YwFrht7WO`*d$EpsQujef;IcD3lk3K&-hDXuLg^4 zgNDR>5hyeK;?wnAU*6o=F4xmC1@cB$dwqOZG`zhL>Y5xKHfCQ+NQK7j9=1MSUa~J& zn#E?jx3)?(ymq?hjhoPa)wuNo4*Xa){_A>sr>q|unY{8q!5xb{^cK6_>4<%gx z_(8+ZD;MlH-*w>D`DY_KyAE{h(YI8Hw4rWEOS|XO-!z!?qU)lw2?ZN(%v`R@uc6() zxxS&)w`2CjdhvSnxnr%$YTNU8+m_$`K)jiwyDziyb;8#2&BCpUptY9H)vEq;gBy|PWHl+RLTJzmfMbY6>fF-wQzb+cJF%D3rM;TOEN-77n7=ghee z@OsUh;IgE9RfYuDP3o67Z?0y`>r8Td8+s!8`w^Kse{;&7F>!$wpXu`kd0tKknP!dM zb9hoid`%N+bmE>(AUgzm~+scnX2@}>&OesddH`x#^Ck5 z&7lv+Pbo%TS9#hfYie}=%N6%uf14{Y^w_#2aoPQw1)>(tEf}%!hv9E~*7|yU!N=zt zU7PG2^KD_<-MQPpk*<$C(R&zPcX@I%?^i!mZ)kYyH=gb4Jh;`fYho^c(73B6zYq^|u|q)lqw6@P!n#u}GS{SsH#sIq&_`on6(!t(XnM2y{g8LtPF zJtAdnIN%{!M`scX+yU!5PX?>D~o{fUNmpW*fOuEQ7H zf3oC8i@8~>p`G`slB?(k%U{IZ8>BXT=gZS%;}GNd&&Q72e%l{1bnC8SOS=!*8$R@W zn-yof9NC*^Sxo=#J^FsKyk7;qbnDJT=U!OU%GM~`%)O^44PS)UhwSBsW_>o|E4*&I zbh5Id=Y%iu`f!*#tm3`BHSpSfcHX7Nl^1=7*F6sINxFXJ!`gVg|5e@%ZSIDL;q{Nj z3QYN-csUWT`&`bwAj0)zBwp{Be8uuPz#c^W*jKt=|+_*X!0`yl${s zf1+OMJp!+b_wCkr#Qhzu@j6>TntvhbH=O+KL43RybbRqagMpb(5#bkrZ4J+Sz0>hh4zTVFqwverJB?3_TvfXCu+8IIOzG268J;5(zI>;>vp+5!d$)qpzg+jR zcTXm5u4Q~V>ZLe0qE7yf;qywJZq{w!ijSLDjKkZvb7b@4y$5;+E%-$)-#SB;ULyx> zI6R_Li&t%nl)Lsx$t}0<=X3^o9&IZASYp??Z>I8dYg}R=l1ucUz04&QU)XI{fWhx1-e;Pr`2=xu;!leC2JE-DG) z=Hd0uO2NZNn`Y{c*BR<{sB^RH*pYa>Z*!k9y_@e|i`Uch*1A*Z`u5d$UH^x%m!czT z48iMT8S{0!-y!iWUe~|zLH5D5PtC#WyWXy2R=ytc6t81e&ge3BZPFIJp89-NhgR~8 z40yf$;xE~1?)d0J8QZ`WAKq?0DOhLlsIJx8ow-qBp;i8&*BsZd+(#?+EwCCNj{`5x zY~5Zsts_1^rhHbhNWN%$C%nE}b&JE;G=BkJXP>t{|D=Nx2H>?CS?{OQtMc5(>n7vw zY%dyD;5=S87&m?6-Njc(zjw>Jy;_;`-GYmCDwdq%NQ;|Zcaza)_W0^?n=hBCSSxkA z?Xd54)2kUq&KWvJ-gj*B-3BMW&5`OVIBn9bo3pdOy1%QDa9x;M=BHzK9(Jj??3Ym` z_xQ{AZSZ~SoeG01R=GB!ioJB3skeI0&NmzHUzd_C+$9!_e}LE33tV|;8yor-uV-)T z9#-ek(-!Lny@D6ZqD$Y}QEzVlQhRgb@2`2fy>@El0~x9W54^hVD;C>!aAteR&dbeu z4EgQ6<9La(Ir0x5&>*x|rD=6W|28k%&-D&hNV>h^Zuvk3b@bxQ#XdgpDnqn8qQYk( z4d*mHSG)14yYQK|&I6l1{5W#_J4b^&+r(?N^X6HXL(Z$L7(IOOj=;7lnJ3jkGj>6|k-q+hN!KTd5?=JSPw!FP^X+a_%I z`c0GkSE?U%=e#^z_`Fia(Jg&X9Sg@eGwpvc<R ze!Ez-X?U$R8El^~Oj^A~i)}E(`{~A|2|cI%RB1}Zi|4XV-ig;|R|LD>_^kcQUlT^I z4|zWSSoS^6HyJ7{Se0#->+NTmUrs%={_4l?YR!H!sdR!AuU6JOb?e#4Q!&q!Qc~jU@yAEjtz(lq zc001{!wvNc-><*()BJB%Ww_i-y506gk*D#`pI`iB`;TKY+}SWT)3uGaJs-w39Fy}z zt&oui8jV|A?32gE8&8|i9e;m~F}BFsY9F>6m@C0?s#(a-m76BI;ytgw!fW4ma{QuF zm1pC1^IrR>UY&TkeA*e;$dygrcAIo#&9qz1Pn8wJ@=n+}CamJe&x-9>JfrQ%g_~df zHfqntysImnjgKq2q9>lWaC|E{vHZL#`0+}?62GQ})qA(7#`b$BC!QKrxIwL|%IZe9 zX6~+@^|>v&cK$~~)t?{w!d~Oo%{wiPN$xN)%Ze}FX52FN`1zPV-MV*8c>8EBejavE z-hRaYvQ0gyTCt?F8L!kHIC09%lyK{ZJgQ%RP6r>KT#!8D3}kX8yUa zhp+F5*Y297cV)l!VK=-kyP;gvh4nRy;C15-KkrT*`ja28ZS5C?tZmtGJ6=EEIptOE zMn{+8^-{;=vggJ(`Vp@uI#!>5dU974yzbpH;=4;jL9o4gs$N`jovFHg&oCwM(EsY`=>6CYp4>#B!m)c^kas^NGYy*liet4D`dcs=goxE6yyZX1o) zE0R;YU+6q!de=NfYIIq+yo^+1#3yyd`K9plnB1X{@9lf{L zUFrC$@GtF3HLjSu=`R2I+pU`KSXTAMjD4=Fjc@MvuN@lpW%mjrJ6&A%?TtCPPJ1lh zPX1wKFi>gy!Zj6I3`?x+4E0*OhrR0}|5R?RXa2kStvuR%FJ5n3yJ1iVdy_|aox$4s zo5T-B&&2Eci^eqD{r3Dvcs+Y#ufxi33o7Gv>5t0}-u}vd0IyvmudGj5^RO{qFIVg> zpPraT)>*xmI_E2UsS*DEii2NFv`+Z77hXT<`9oBdy_tLB_2uMZFPne&b8h>J@)Mpf zsu!sF^#`ZRc_$6`HoqHDyw!m(w%)7Ot>EGP?Kaq_^>G$ywsz>wGqau5zqx(nO2*gq zth;{8R_fxLmdRPK&0XZn>S}YQOqtBTSr2!xG%2t-u(M;&eYeKGSr%4*@|WlFrDYfO|Lxf0XG_*s zOLQ+>wW&e9`JeiGzp6avu6JguB9G-I&sLwVb)vz*uUs)s-=QxBytZa?ZfQ2JNBa>Y z>U^igRati5TD)wf>S0Up+Fmy_XLaA>_SNq#n6~(r=diClOTD(Tb zDR;xhZ!hz4>YCpUe&y*LGv`UW2e)sX8S#DW2Vc(W(criGE4)80AAftmh^{$*nK&?R zRhBun+1u*$uKZEt!8;dMUh4KJYmWTBOh4EE{6wF|#_j5#pZPGuSpSnZ%Vy#AF)EpM%F)@kfmC?edE43iPy~^ zWPBIa_fTZ1s#zNP_KGp*+8sH#c>CLJ{JP2W_IpRvZuqv)mhqYUjA?qE|Gs_fe8&$< z+y&cLs$08uNc1uOc;WrUmlxjNM)}XLFa4rQ0I$!S_C(#-J@7bQuPgbu|_}NIOkxh3DhT2d`(g z*ngq9HTey$_uOw;FN<~HI=n7=di>DPhSzvLe||h&{L4vUy5_Naq%)y@zxW9IO*8|V0m(}t0>+w{**`_;dLi!FF)!8?#b+!yu zF0aTqq~Xwzu3gs@>9nfqubnp(-rwouBt6HjvQF-`}s1=hQ#Co>y<{$>EhtzQFbW=z3|9qMKd*5MQqc zyxBUo|7LYatyPIz%AKlK?)33Uc!&1XB7B_j<%riWqsASo*tOyo+z;3D>p!}k!so@O z8I{4ACg!h!*SH@(FVD45avv(4{TS}wxPFBihnAO~&ad%#)U$5M(nmM1`5xEf+p+hp z+w3f?Wjiw|GyXnaKX1hLVbd2( zyYRks=%UCX<$t(24}TwDuV*$`P`*N@8lT%+HyBbZSa}luKA*>HJpZ%nr#>E@Yj^4& z<@xXDy;o=cymD=J4}Ftunh{^7KE9qEztF?iXaBiw#jfOz@4Wo-1AING;cVW%$?__7 z{%HNa^PRt)#oK@A_>9=}DS+mU~E zjqgYC{XV`t`1{TG-7nzVeZ2kr?LIz#et1(Np_{jMCcM4;^%I{D`1;g3x7~R7!KiZh z`Z49{JY{&1Y$85?zvz;=wx#vLN4Vc#y&a#J@s~=;KO{`W{l}`-@Aoawa0G8Jz8%Mx z7ry@Fe{-kziA%$-UJ)OUnp*XFLH=Z7mmc8++oY-Wk$C;r5qN;T+wr`AY(yM)g- zd^?ocf9dDvw(siU+Zn$8<<(LrzM1}X%m3E;vlVNt`SG{69cq>>pK`V5#uxndyGO4Z zRp;rc%kSHMe0kvW5ntZ;c=KzVUz+`+lB1vWvf(wZ=g%Mf`0jj#JiTu`Jrjk`4}5)i z-}m|JJ%0X#>+u@jufJ+>+f%s8mTx|o(qYiKh6U9NSsTr2aJZy_kH_RWt$P=+9&U0W zXQLO@t0^`4 zU-)?7+mFAx{;Tu;YW|I`^4?Fv(8@Wwr*)!79sS% zJuiD3wQhfhImz$6f8$>d!E5j4-eUGr>oeo~JO1ncmihNu4VW|j#FwdcR#g4&XhO^h zeE*K0m#)cEyt3od;FtLR9N$0SHQrx-eeTOcElNI1s>|;;UgQ1$PtV7ffB3rfZ*ZjY_|6g{uUl^A7s1(27c#Ze}KQ*7f{rY#W@$Hj!=FasI1D-$U=i%FT{`D;U zybpgLzaH{OuYce?Uyon!z_+LP^$ncIkH_#D*W=fr|LW@(fA;r_?JeYsH@PB=cU*E&8 zx8cX%`1K+F@i%|@E~J@ z`0~QHKlpg#$H$w?ABZ?wxNGUAGxqA-hRA2c>TWD0r>0X|F+u! zd>rumEcm$L*CQ*n&0Mr->N^L1KNcSse7m(N=g^pbiPyiw#{s{-i!T@4A3yKlkH>_s z7T=n`s$HE)mELa5z9au5-2dU`rB(NjF0yS*iP5V!6ps7=_vg&cUzoe&YK-ynf&9 z39iS-nLjT7>hlD@u8ZGq!uy9`2j%DE_XEnca$op*bm#0%Lb{{{XD65actqJ|pR~x7 zh_6rh{uJNezVG#ZeE;*l`y+h3{;wQA{JD*(13rIpV%6zhe;>c}_hs<)3tyi2@ydVt z{KMOW-w*x!eEj?RjNbRcw}<%p95N`|_?o*nSN{9@{EuCq@%v(bAHTnL{Qf@w{@U~3 z_uZfTeg36?jtxI9{Z&;M*5KmUB+ z`-gwEAO2j>{>9VF9o<;|DSjQ|b#42Sb4k6nP4C!${rs$QOZ++nzJ0}O{QA%T)S7>t zgC%$^_QS<87|M_}Ered9ImiA4% z`U`(M^!CNJ#rp4`@iRQ>F?W} zzpsb?il0;azkWOP_wCES=l11&#}z;Cs_-DO$iM;jhTz9L`15(EPGo6v;cE6__`Kqu zcfIfN&im%!$6NS)JN$ENa;GP?N3Jh%i=RKJ+8193Pkul3Up4=KdLI8g8y|1{em~w{ z{C@tL?aJ&A-Zpr}?=L={`1s-H>G*RKKUd6u_?MfdoALepiM?xjUc7zszJC8~J->ha z_VM4xuN&~6GsExu@ShXH?}Hy3U!!aOnhSE`_qh|E*L`=Sz?~~(te*50{^WV51{@e<#|D)~svp;W*`~BJN!Jliup98@^2Z=wIcjWZL zGgp@Vx(WYWEq{68_4__Af?xMPa^pq4uARnP`8EDr!$10S3;1|eFV^&C?}Zgd;_bob zJ6^wU`|$VwU3q+Y{HtE~!u|Nm_m6(=6#v``KHm8E1>k<~JOAu&3Z}jJaQY7XbI|yB zzwhV$`OEj;eSH7j{SO`;w0hT!l9@YxS#wIiEd_J@_``35V%>XM;_C_ieCD{x#YYF* z%eTYaYXhS$&(dF)_4m8qH)?j*efy_9TUF{he>;04X>Ldn_@ztlyPd_y8UMT$zTEKj zB|KkT6tDptqVZR^9XOQ)1u;;KJb>AvI5`k0hCqc#*f?#+Dv+>FPbZsXp3p7-0j zd|yZX(09M}U-Cm4VR6wJbwUqsnlXOX-fX{q#pm{&$e#a5@g-dr-5>7EAbwRCZ&%rJ zwI}AY%vyVF`P|#zEEv*0u3(WCPYSQH;_u_Hx4TV#wPS%?JN8ZVwXh+juVQ~UGy2}v z#b*-m`a|cQfnm2Vbgx-^+q7Ce*A6LIf5D!|%LdlO-^X8D_kHs+t?<-oc>T|Q9oBl{ z()PDMIrvS>gCPOi%pLx?tu=PlI$WyS`SUY(Tz@_2&-P!o=;2qEV{IDa<2w0xuCrPj z&-9kom7yP1u0Hn9_Q5%PIcC`Qaq~kJ>zu=V$0YpRy6dbb6P~YnQ|nAGyU=P$*womi zjSgoo-C^XgN9% z-LUQc)VNz~Ci36^>`?Eo%T6g?hxazs7T)@1h^1OmwtLSrZaiO!|9;-lmu?;!a(5c< z+3FwL+B5Zh$*qx6zj))6S*1U(vq2wFwQ|QP zh1YfYNzNSXo3o(&<&PQaEPtJQN~SG2`gR$6d2>qkTnBD9`uWN3oUgOp3(TlpTqt|G z*xu~HebP$<2Q4+Sqv1JWuJg&(ki1&hd7vCB1&-$Taxl!F8)mznbq>`!iRb zzM8r5v(%&y`Ee<1Z>~|!cu=5aode%iyPRv~ zjahR_4QZ2TnX$XtgnRM3JtxmM9k4&+h^wXEZ2PIe_lnMEnZu;!SkW>yF4h|F#qa~`({>;&GGV+i)&Vdw74pM zKC;~YOZ8LEJzaC`gQMXW79Q0$7tS2gu~459Gh)MoFN_DHYq;81-2QW?8>f2re6zP$ zyLC6bBg*fr`e;C4Op`I0k~cm1aYl>DFNfD(w`>26ktx3x`u0_H>V#R>Yf1g)Hkwps zU6omd%ZKH(_Ii-9>DKz?s2jsV-eexw}Q`!PqMEYGfYTzkf>W;$!DzG4hLb^ohE5q~_N3FKT&@Ol?}I z+R4bgQ=;;oJ^HNUfjpb*jn0^<$B_diUQf7jzNw{H+Ti9#-7mJ(dN$V9d&MuUW^`S4 zw!@2)3#0|bR($s1>Y~MVecxok!eh%9?YKAM;)IpZwe*k=e)Wk`n9!C5Wt5_G8Y zfUcbebxN0z-!nq{td`|jES4?#kJL=}u+Qo%f}x3M^?Sx*iWJF<>fuF8CIbJ*k_r6} zY%tTuuoFSw&JF*fiKNeR2(d&^#s>dxo2ArUCp*BVuL@8yphA-~U-GE#UvT80iQS$4E4YpOFC| z*37sM>MRPPBa#Sw+cE1jNUP3nYz^&LDbrdR8A0 zCnG~Zq8agkBoKki%g`0o;pauS<-ncOGn%0DB1Q2cClG<-V3`850z}gL7=k?~+u0Dq ziJ+=EP$e+(HHe#t*+=mbK~)KmNg-lp)F2U**$SD^TzEQ?eISuU%qrc@i;+YGRUL(@ zAd%mzR6z`f8?*nq?)V`i5HT~dlL+c_2{Lg+%#4yi1Z93gvlw{<5}t>il?LKuBqQ9a zc^SzGlE6qI5FaCU5Q~-WTmeL2L;-OVN#ABk7*7Okb4|#2iI^EVkqF8}LMDxfnNjq- z^z`PCaS<^ynuixXo);rX1kGv>eeC(@&fP(x8F7OoFft6p$H;h)WFqO?qo|?z>CQ7C zV0glEp8OyYj1&inVx&As0wW?wkdbO2mO^yr zx*(y9GzE!dqz#ChkuD%fjPwObV`M0ZwJ<$v42X@9sUR*!=7Pj9vJ}M2$XXCTBilgG z103`P{j*s1!xtMP$3R?+oB{DNav3Crk=v+G5xVm)AOa&VKq46lfm0xskz62&j1&e* zW~3BIXi<7rMG&146~xKN#~@xJ=$axZh9j93RghD7QG&dvX}oBmAK>lzy|XGf?7Zj# zFGeIMq7cQ4=;lR=mE_3@&m@Zt#aqFajL{X%EUr*I6djC=+X$w&tfc#uL} zyXS*=n9MSeBu3VQ_!-#&lFG%e;)_0r4|Z1SH5v8IbUjc>3?R&kk905EY35tMO2CXJC# zKnmD#XOw9OVq>Hwh>MZWLEMb=1W9CMAV>-$BSCDX=vk9M6h>x(=!`4^iD6_Fh?kMg zAbuk0`>u&;wCIxTa9B#?>1IaCNyGyG?S?)!Mh=6xh?rHXhX|_r4l>C^%#7xz1*Yrr z10tyEB2=XjF{^aF47G)*N{JwXs%}74BoU}W88wO)Lw7g?H!UcmCDDR1VhSxNW~5jmDC2{SkBFI3Q;49<3CP$g;OS;Ya}hzA^N?{9F*AA`5tO+GnFJ!~ z8BGWhK^gRHKD;73OA{lBpv)7(xh-+6rC3}h8I1K7bArehh$aQ zSq|CBi{j?R;o(L1@nZNn(S;OVWUINejxTZeFw`TA;HbrSc*R4z{zQMf2iFU_}#zWL`w8&h*g)$;FEr&5M@6i;={M zD5TM17=j>J4Qdt=(Z-8n=S7X?#W9{2!^?>%q;Mjsma6P@Nw@N1xHyr;7+&OfUX&ma zbW5@rZZ~Z3H52^Hh~D+kB~3N#)u`zpnh?&59L0+g&x@ACi=NDj5n7#{<&eU8k)wIh z;(2idXfXucA-tvq_QIiu)}UsgjBKL?GP>xZ1!a^tT2Myv@gnn-<6zQUWcyHNeqNi~pib5T#GgOIk z1TTt*7d43&-Or0*t;=?nqzGPQ4=ZJuPK~+!T#jt(KcGd(ZEvSzk z#fuS73v|}xBwC=#A*RuSGK#eUH49~AofjpB7F6Yk=S2_Dg8DeD4Y|&`lNRXX5Et_z zC(r_YbTLQ^s#3xmv7L1(iWZbn2hoE1C>~l+MvtQfWwaDpP{yz|=4Lq}Xn~9_MbiR( z3?YFQl#zq9pp0m5!ge;KXkO$PT2LQtJTJPJ7sF2rn&k*>O3i{kh7d^$%IGdyP)3gC zMMM`UqNmVei0}yQ8831gEzn04^=9;Rs8V8RK^ZB5 z7SvfyrUhlx6k1S5w>76aqm1UF1!W93C!!R`i=4uXX8DZj41GjBlounC6G@0+#E?YQ z!-yzIaxx>3(L!6Wvm_;)7l)G<-OGtA?&C#H=0yo@$?XYw5srhDWD%UX&TdZTHrVvY8oTZN3ng*&JraT zEvU*7#|TuZNwlC@a)1_8Wmux9&Tua%2$76HAKgO>>ZAE+L3?UgI#GR4#$h93f&U(6 zg;yQnYbN+-J~uZcK_{8tpRL19wucubnHM#*GdoLi*lB_7kwk$O*dEDnav}>}T2Pge z$cvgv3-pm?YZrF9tT=gbxOveNIZ?zkUL?IM+gVYZyl7FpIN~``1s^Y>r5oE>RqVW| zQM4GMB!~&TNXfj&L0+^pUL1ONYC3cl1s5-394)#mNTQDxRHayZP<>EFa?*k_Y7{Lf zBYS8;8O=)z$|&)?sEM?oD#OQ#Bv_)k?a}SL7*Vu9A4!g(1*Xel5-ljBB-4U2aw;#1 z(36^usvJ&UbT2Kak0X&6J(U*dBa7BvY-d?=@*>C30-Y5xi5ApHNuvd2q=?>JXGbh8 zD5ECRg8Jy8eW)sw(R5yPCoNE=h*7*q@w7l6Rq*p7TKiJ7P#;y`MRU`F`Y4IKsHwc@ z)_zoHRAo3h(WE$DIx2jN(O#=SB7LqFLN*XLy{zi7v$QBE=Jd z8+%L5P>W>)5p+8*3%Zg-1Z5&2lSahMsFwbCI?6PMjGdA8AmK#JDqY~ka1%j&xL2MkQhd?WP^AA8L@)+87T^q!bliMfRV}|K_altx-5!HDlclpU}~FD zMspHD+pI&MC?aMbhldEt)PjtckwzeXMp~imA(Gxl5>knvK2eYnhOpZsi8>LK=>-`l zBZEL(M9j`|6fdfq2!ce@0D6<$Ol8D*Ia1ue8)sXQL zNzcea91)aBgiI16dq8|dpvo{*QA_5<;im;IL0L}WMX|@w+k6B%M>BF7B$i0}EJ^Sa zL0gyt8QYh*kC_n#A}DhcGEO4qp6V_lDDwbKXXIB9HxaY5;vs^nEZN~baUy0$_0t0T zu1J<)+!i{lM1F4reQb;r0MUt^&9!NAJ=qDuHjI;&uGSU^q&qzOzAR{p#sf>&TNn>Oh2>NlH5cq=r zSuFEFLW!gw4Mhtlf->=t5g1tq;$~z!NGuVvkKrYP`W%2v3X$}TDg=n2OfqC%GjbLr zbR^zpvr4k^q6kD#pDR!mL&U67Wv!n083B}h0UnR39kqD4jB4(BB zC4#CNK*mqR%qRgODANKm1w8cjbOb5JNHj0AzF`W=5(*3mjG5 zaB^Y@ULqFwZv^!55i$Fy0U{_f5i;Rp@HU$nEt&|*cp(!@1Trw)kmHG<%tFWnh?sr! zR9aA#VjYWTp{kWoRh^MdAa#hO_t6C>EvSzcM+Egrf~xIA%qr1G1Z56ECXI->7rJE} zy*;NO6G6nx7)~On&jrZDGJ<|;DuKxFRgxeFiJ+=`kO?1;XPFr#f(XhygG?Vr-h#Lp z3C#)DTt@PNj9{b~h=-AKAmbT9KeOd!q$)@vBXvMh8EFC%Isxwm+GZDsm66UM1sLfA zVq;_oh@BA+NCYEOKs=1h0f}SeYmh`n5B5+0-h9>EWM9`(9K*q;N zO%O{g-5LEvT?8Y|QD;WlgCsH19VCrN`gB>*C*#hhsFA#AE+S|P(NFKWiI`OmFATHp-Vltzp|mE@!as&p}$7F4ChFalLd5+jgt zB-4WW$Z52oKDs`G+CsF=S_CbqO7YNwGLCpg;DZLjK1Lv;`)EOZ9DZ70I((}0D|R~E zGtz>p^k`a8l^n~9lEjFp=u$E-a;TS@1)bpy3|dfU&CLk(QDSI8Rg#w$RHdgd0#%xA zCN&FX41pHZM~$WhGP;n!2=vjDXhBt48ZB^H=!SI`I}6^ZVnkF7A%YgvM|aYKrYlj5 zK$R3vi!Q@EEIwLLm1>zyO-C6~rvEhr-;@gk?u0#%Y|n@?>Is#09Epo|nvivb^Yc6fQwgN#5Q%^pW} zMtxMB7Bowbp#^0$A1yFT6hjwKeNdGV$p~a5HzT4d39+=G&PE(9sI!v72vkX_v_O?C zgf66~LzQM{1gexMT2Pe~PYasmNMr=6)D&Jc%OYwP>Z4eBQJu6vXGw4~0-g0lMj#{l z84*=kNTmhM(uKv;bktdPGXhm&EG?+YNTLNU9a&0Y1p26{ylBD_dKOeEE?S_oB6)a` zeMHb>q#3#3vHI6sAI(VwW#S;?C4wGn!7N8S5tLaDnG_;sMh_A}nGKLhV`L{t=u&pN zB18~DRX)f>F>(SVj!60}MM@!ps?I~k7SDB-?L<)K8f2VA%sz^X7cH6y6#f2tBVrkO z0)3K+n0@qAUJPLwyFIE9MFbjKUPB)@BUy68a~UG(omDA71XWofV_A+nBPj}EC1O@7 z5xl4#UbI9as81O5@i9^v&02wHnVoeT5mco^MkkV<(S%4|#8@Jzsuolw5izTzG$JU| z2r|}{>~u{Lh@ea>$he4@eY7|tC=&&l6e4EEVOhm>)^#GNsuxs65=pOu-*iL-Wd=be zfry!re7wk()p!=F8UAH&+ zG)s;qf;uNc=L8~VXC;LZ=q!b*8LkOn@bynT9pp57vf@bZ3Spgzw7IaqZ>v3n4 zIRcp|B4$QSAc8WdA(KMH+-5z<2z1uMH{i~YLEEE555gPtP?c;=q-SM^Di;y6 zN{?X#wor@b#S!2|58Z-0LmxCt6NsP+jIl)B>>SILHf~u4lMxaWH5A;!@X+c$zmljmzNaRKL zGXi}y+jgom>Z69!0-sA0^k`a8MoXZ@Fmyo-FalMEeFxPUWuyp3L=8T(L<{O{#L$AO zlq5!=O7hbJRq&oKEvQPf?xd$fmEvRsG7c{zkWrIpL7n9wEpRkMEqoW%8D$hFEihda z;uwJ{J%tg-h^f3t_9Ut^%#ws?Uc>}mBtIk2Sxw_b(|2>76&E8=rAIRY(=`t-4j&`% zg9}1{5vbDbd+1q^5hEE9HCb@;B6)a`eY_|EUbIwRbn9MfI$S!6=A;FVs%*q_q6mIk zP?eEF3sfnRZ6Dhio($81GK!NH=%Y$rT2Mxf=SB0=f~u4NFRFb%H68WQBREloXj)Jo zJ%JXeQpF%GC}Y?Uu$@&Yf)~X}3sh-R3@>sLEohdO!i&RpkeY@1s7_i?l^#tC$~ah@ea+E4=!|NNx~65wl9M zoaXwd5kycQ8&tU%DGd@w#O&kn5kXZSK_-QWnK4p1fgcF@9#2PA8dOCw5&;rVB)yL) zr|_cK{B)lNP!+*Q3lI+xvyUU52%6pzG65oHMz@^7ok8GImX!!v48sr&FA>xy8v3L# zG62MOmh0n)qy_p&QYAwMT;SV`mBUZ5|Q*ivY18- zn(oklz_U=*Ce)dTStTX%A_r)JKB^FYf$oz8eWHn^Z=otC5J9sJK_;1qIZF%j;t0Qp zJDYvvNFu1uDd^)NV)ilOX+c}4CJ{kZ7oaMQh}l_-NWrsECIFc@BI%tKF_{-B^hdht z9#rX!JOhbg%cqgkPO;abUL@_{5VQVhg$30I|WGu$upV#IS|2mxNCG+va@ z%WP*ujpRj-=EX>01)i-Z^CE{{VLHPz=x|;f(VU2ahZoVui=09P9m8^Pe1b%NKR)pM z)N+-bE@}}(P?Z2xF^p6N@i0;c#LGw%kT^!rZx>Erq%%k|BYi;pj0^z@GJ<|5vE>@x zX0(M-6?q0;$f(Pau(uJOYVj zBn>2yk&OA^6I?{nFE3TL2k2QjArnQ!%qVUmaCj_*AQMZ(%xFnOPzL?ZWj1c@hNRvAe|P*o&k0*o{V5pLpHDAOJ!j!1eRO-kTJP9=h>xcB9*1TzvK%CpkqsciZQR-HW4MT* zK06^3MX2K}~d`yJc|$qCe%k@FyKMy`P*5ivWYOD%+>Qt7KIpf1+liP}4mS~0RTMIQM#4aBKhu3GgE)z#_kmBC z5J7!($RrUlGh!MMl&J-o(0g>}Mj!$stw3Ukn0@3}BB)OkWRi)P8O3s+?%WG95sVB1 z@iHgLh=-BIAc;iGKDv(x+UC`eu{@+ZCxV0%F{=z05mdDY zGF~F-8F-dW1Z9q(&X4HMr$O9|q<|z6G5g5LL{OibkO}<-_c1eyoe0W2fQ*xoUqRxC zm{nQ=5maR<0GAgd*+J5Xm{q!-il_fx1eq8{N`S->F{_MZBB&}HGM2}59|6n9R~$fBJH z%EUk>iinw!5{RJ8SjhMpnFeBg#&uS8UNjF8)Mp-4`58gKLErwI>!U{zK~?J@;~|p1 zg^G~Cix?z=s{NUKvEcK0TN)OBS?^uXprz%^sE6OPDVz6co~@p;$y@M z;%8(bNE#z6LG;)3tW6*>j3j}0894-!$jB*>G)69fSl`f{10b=C+yhBwZP-V}`&Jx84T2PhY}o)$P_lIr6{v*g5`%~^__7F6Ykq6KwU-L#;L z5zmP%Ch{Vs@*-PvQPZK1tcLUAh~P!{@M0toLC3HV93LN%^y8z5mfUz2%Ge=eC1Pe| zoe0WQfQ*NTnNgC7po{{U03$U)+m0V3%cc%3#cJ!=?boQ#YINg-lZiK#?T=NXW(z@M=DuY0sOkVqnC zl^jb1RV{~193vY*d_>GDHAoAbb($QWpPsc7s@z1(DlLf?R0V(EimviORVpJVKw=8u zjM>MLLBa(#>lA}AAtjFXWkAZ{XNXFZk{xL#;tGA(Gj z5n2e(LY-ejAAyl9HrOvB>HDQgNkmYU6*7KCih{Tbp@Rj9MW z&Wj#J1odeIeY`}{`#8jSA}G@eGKoYW1D$mtjTYEK_-)u9;OVF;3aX-rn4OgbT2LR! zM+8;%f+`FAiAehX6h1FX3#yXih@h%LP?b!?oaIQN1y!l3L{QZzsInI4W*JUeP?f_= z1XaaCm7hrZEM2gbp!%RHJ%R|TngvyEB4(8sAc8WBA@iDunUO%av>*|b*#jA?9d|Y}ju;{+a|HDvVrGm)A}DhjGJYcI8Ti|jrEq7IL4RY- zO~lMdNkmZQCS+2Hm>D^yG~MR`Wa5aJ88v|j>hmkw9wKJOVJ}1Xu@r$blZcto6NsQb z*&!1kVrGn}FuG3x$askSo`K&$7)u28DFK;yB4$Po5vPPkQgFnl@Ut>&AJ1bL`G6Utd(%* z^eXrTe?(B#OUSqx$y5|xGbdtJNohn-Rc^=#m2qbzHjrc@W|bUT1!q8X*^tF>UZiLu zs84C=;~`@9QN2V^<|D`?GNOT41#Xt&B!a3UAQMHz?Cgjkf-(&t<0E2b^iYxOtO-O= zRST$cGtv>nB2iWF+cPD@MhiL`jtC;CPc+&>Mh1XH6G`75Sxg{;szyM@$H+twn@mlI zS@3BMT3~x**+UD;h(02yGy3b176tb~vJfPah`BvVG!c|p2^lXDGo#v6y3Z!aL=b@t znyz?=pgu{E@iB4;BtXRM?67ONv)KoJCKU_na|)`Wh?rGI8WEJafVRg$R|P;~iKJI4 z@Rwq6#+;=GiJ(6Bpi0nj2FWuJn}Nmb1HY4%6149|1QAsA7OJ9&m{qce7N~+x1rtG4 z=x=iRiI`QYwJJLc{yG2=RFw~^qKTMQnwJ+x0xf8|nn(ooDF%IfM9e7)J~0Vs~`iS3jz@|YYOy5 z9LU5IF*9nA2+Dj78KE{c3oZ+X7DEJO5>Ou^W*>*G4z5C(t*8$XGh;Z3pv*qVL=j2P z=wcEPlsO6+OI_UA%t(<$P{t1#HxV-<+v?#el(~eqhlrU`+Y>>VpHLqnX2xNwPxpBQ znJ6M=2L8$kok@d?kBFHu0+BeAE{6Dw7uoVDmr){kQC&pPHfJmbw|+#-&W<=DD3cR1 z2}I0{5g>vxg&<>Zz)Xi1m!pZGj2$vwBER>6mu_thsVW`*yr*I31RfjH0-xX%&FDVYKq862(SUP9RicQX zqY;FRn~^6Vu|(33p`s@eK~=9I6C`403`=u*dY0mFmNQ}n(TSv2sgjEp*g{Q+qXo`h zRZgGPT+4_@FFGfqWXE!tSzWnusse< zpar&BGh#S#2uZY{DkDG(R5_%umTYHT2&V;Qln7d&5B!umEhwXUX+aq!kry?M7E}d) z(}bFi1pcN87Id~0h0EQ;NEk>mBb7k{jOZZYt?53sK&ms+2qcn`Rv=M~M1dqQ(hDS+ zkwG84Z0usr{ zO^_%?9)Khe`TY!+;kWbLC^SBoQ(DD5<=tLVG+5^^u^;$w+mOC?aMbJ%I?Sst1`=B4);LcEFv} zMUvdS$cenDM|g20^P-1##GTP}CrlTJq|cH?4=<9B2&!reRRKo2f`orgckTxg!AJ~< zlaa9?aYSIc4nL?Z`iP)e(;yQXMR%SDVkKf$DRy4eXdh%IeGWj?K1Py3ju0`cBtH>Ubrv$Ij9dYU=*-PhyhKpd9mu2* zF*~caF1QM1QX$ibk(VG*jASYauaz*88zhm4+1Zgy1a-DSCWVpGAORv~m66H`Ja6hs zZ{bH!6~%~#rZW-&lEz2_5PLV=Cw&(Donj(r3tK>D4kH~w;)$45ijN4YiiS*(kpUp7 zjEn$@=#HnOs)-=6M4&VLh{EvQO%5kXZ+P&JK+ zxjkwe5tKOunM5My_UK8x7-_ViSq^(IygjJTDd^)SV$RZ&h@i{`$fOcUp9Mea-<$3e zKz)dq899a)m~M#iL{R5@s56oD>4p^1hwl6gGEt1Y1xX-c&eHs}pjnEwFWo294(B`( zv&s=o1Z__~$aonk1`QfamZX#wz zOd^6Zbs!U9qzQ=q3q0%h>F|uzO$1fBAd^JI>?{Xp(c#x-h-pMnRcENOyK!eEeL%vA zn0?epA}BKiGF~EPMoXau%~GuW>CPUgaxpRm^&w*JmzqKZMp@=S#xel+F=rV%Eohb_ zh8FnKd4k}j1!W8$C!!EKkm(G+yQVITqu!@h1SNvbKp*(aytJTMS^_O7^Z&TI8z|?l zsSn^$#h?g^pa_bn;(ncb&!sA;8Wd4QPy|I#1VwxYMNkAqPy|I#1jV2Tss=?+1VvB; zMSKUv``^z!cfb4W^{%dWt)Kt??{jbFnP+C6Op>gnP$sLKB6d+G>u9l4B~h@iC0c2T z8&CC{qqL(Xn<|Next4I}X=M*?LglS>d~A3!g$$lvW{Srvjz=ss^$}#w^yD1KSuAeK z%vosaJY){;m(3&#NR`D+rEM0<{1BO;Cl^6xp8Nr_^<)b<>=|;`K{v#edU6P);mI{2 zfhToH(~~10BTo{@(vzbh^=HcMyO6FY$3yy_oCq0s@+ip2lOE&(Po4=`d2%YG@+`Ui zm5^Oe#*mXec?YED$yt!TCm)9lJefmAo_rOu^yC6a{n_$7KY{c-SwqhDL!r#l!k|i{-#)61?upgavn|avfyWEG}cJ&y_ugBhzGYnRJ(hdTxcxz!M7@ zdU8j|h^6=lPtyqt^&E%HhQ;m5?epY$?nk>k=|C1N#S~t?Y9OZ`i%g5fZO=EbeGYrc$UUn`^O^7pR^r&|>>i(36E@EzwE}KGWhWy%+k;Hfl+s z9y69gQ}Id)Wzxo}s>igYP$nNpp`L6m1sS}8+>2C?ZAzg`JdlDOyq=yE$|O54_IvP@ zEh&^qJ5s2}45U!T?n|Ley7LmzOqsmNLLZr@;Nz#~$#WrdPlk}nuaPJ-;bLhgD3r0Pi@QupKqkgg}EK}Mdu9y0c1 z0-1U8K1k)2a{JkkT~9s>X?d~_nXnWe38~qz(4+HhWbCVC^M#Nmi<`0|7Ml7MGAkCB z$r~ftvq9!4mLh}K1!bY0e<9OjahZ6|LYadP$DMz5+3Ye&lZ7%@MrPzm4KnfMMvxhc z>q#rGDVwS1NMx#>q>vU1rs(aUgSHf##F?E)p-i}zLYch!TDhx<=D?G?K?W>tSG3PU zCp-a}jVBL*)J`v(i|fMoxhyo*MdlPwo=m$~+%7Ycf-~EA!9qRH#?+R@O{F`pE4NeT z#mF>089_##ya}>oac52|ua`Y%BGY1VyYjIVI$^eyf;(@6${S?Q2hnq+Cwq{lCtrXJ zS&H-I!Hk8j>wIJ?W7+&Y$SzB9Li|)j3Qm}ZV;1UJVQR+Wrjn%;nu;rLl)HY5sj4S` zh15OyFQn_q&Q0*ykHziE#!_gPSxKRLXd7=Tx4XNMHd*K@u7>80Cx=0%p4=2N_auN+ z-z=K3D-YXJ=sfv^g?8NrQx|!13}nMnTvr}d&k)Tv!S~CSg?jFZsg5UY$jp<6Ln?0( zJ#<2|D}|cl5eqe+jHxAyJ5RcmLQ_fb*0PzVo`$KmCj&^wlb1nyEUr1YWS&9taGy2T3xj11M*H~!knwaXaxJ)u-p-df_xhF?J zD(_Ow*^U;w%R)T~rUsrI4cTWYwnyQHg{F3q3Eu7RiXzKGnd6aZv$&q5%R-qGkr{dN zD9DP%O{IzONwMl96xN=#ko$r!Tn zyWM|yHD#Cq}#NYj%gWaP=uAuCTVhBQ80?!sNg=ff=C z)snZA*r1~&>}rYnTH>LWWUR%^w4@6u_#AGNWASKT`oMq1M}p1PI%69j5!Z#OJZx%- z23q2kmZbVo(PQxr@kvt(HD^67cE&;N^q6qQLOtI`X2aq#QRUO+cFJ6cOpV25;yMdueuYfKlMTdr@-Ik> z#r33J7V0_p=J?Kq#bvU67Rp>1nUyCs$j&+P%r}BGJvkE6VJVu^pwB|hDKdL3E)yQU^{+gMSl3wWXjr3_4oE2@CZ+96fVSPKH#!QtooirlloaXvwNyRa3U6C2vWg6XruH zIo^&nno2R~)ysBx5bM;aW~?8rUXA|`=-Au342Z@^cWV(d<2;mOL3kgAF)v89Aqlr z@_S6sVxi1=$c$Ku9uuCgCB8sQy3&$$zO8oI2@CC7VAs@>A3|21Tm%WeDPfNb=1y2O4ez)93SNaEBg=IN#SMwn5u~24<%#6il zlFIjF&q24q*L5r|lb*yvJ%=FE_2in6i6?c)oW=Fn>i6ZYBamr%l0f<_ZYtkXg72hQ z=sZVbsLyO5?Q$3sS*oCuk+6uZKBttAP5Ae$eBsh%f2$iS0lLKdE!3aS23_Pi3( z@?;F@dGZd(%#*VqD^ETSssBhe&mm1uz6u$5asg!G$xk3tPu7sNCx3?QTqt*40@-E3 zJ+yeUZrf#{N9VFfV!J0*$eg9P^GUqYk~Dr??s6H^WufNlpl8fdY&Xf0g))aDvu1G_ zQ~8N%wlyiZ3X`^3sOMJb>9M%;q zNqQ_absVNfp4<InVpN|)Hfh4PnM9GCqIWQJh>QB|E28tJEZ4H<(BC21!rjLD9DDTn976Jul?q` z9gC?Mi<^orVWCV5naW!AB!{z5=7GorEUqUVuu$eCWELzglWn!wo!^wZXzB@=s3Z@4$cV*FMJp|F@Y`~i%Otxj)O;FxI-a~9GGlR5>4Jr(CdjN=T*g*@ zr*`En7Mgk=rUopE zw)s~f=!wIQ6za))QYaJ6w8Z;bl9d*-m4fCZuKrDJPltaOLYZVDg|=sNDU>lQDU`AG ze~2EE+$zDpk!cCrTB5F&c%UU2YcX>z>57FO(^`+n*=Fvn;yws9hqQzAH zCw5U!T9ty2I=maB6v|{>Ew(QOQ+TI5E%8uGvhNFC8E>N{4k`z^uRspmjW94)OWJ3l z`@DzyJoDrWkjkaX9?G0gj~7eP9L7B@X2wEO-^0`%i<`>UER3OnqYkVc+$<-j0gXPp=kg6v)g>+bo^F*e0895bDhQ(#FDGQzF zHpt9aTqfVStn4`knHEctiGv9X_1qJgl_zb;9Ek$qcAmNaZ_pKYULDVddM_cT*h`;DDzBY#w_SVvRFZ}}H}HGXsHG*@ z(_-dYvaJ@|tozMbFw+w5X^A&llHi7_C!J`?rdslq5*yTSKuc1)iOQHjOV-k2N50_AH#SuqP#t;~<%7 zNozL~Jv5cHq|j70l0q3XXQ4lXYy25pu^jj_h;Kq{Lp5i87Ml7qriLuV6rS_OLYYgD zY2IA*k~JHs2AM4vWjAGbywyY28vz9fzqAi#v~5vCw($hfHv*a>_N^ z5esEHbV3%F$s4yWrzrDSIw6bOW#=rEIRzPf16KL}f8&VpML=5$T}3`-p{eI$YR*z@ z$6xTF=)qJRELdo2h^cB+W=LKG>3VVoWWwT_lPwEPy&D-D%bppe#p0&SR0^FZUa?Tm zr!ZAd%AVr>#L0+-GUp;QV{y${#gtQ&`35pg7MHOT7RoH?glReDc4adb%KRLe4NK7z z2klHv*{&4)d5^;p3-w%#o;i!_iL2J{Nx~KjP5m8HBNjK6%%tERCQ&sPJ+wUyq);Yl zvQTp+!q-16t~r~tP~-~8tXYtup0siswJUE*p-eWAg7YN7LJDPUaNDwnb{$IRVJXgo z*Yswg%=M9}992#g=fQU>QfOD)k%Cd32Q1XnLQjXKxUMvuu~6oL z$ZS|#CaO05X1pdd3r(GbsR@glG8-1kJOP>d9aK-&)M5uLG}Xt{!jl(J&mH}qENp3s zMl95G8m8u+ydF}$lj=zVEonzfHc?`Ojg~Mt#&5PsON$vP$)l;3WW_>PF~ONPEX94! z)8NkKF3P+QnI4PFWHT1ZoQ=%Nlg~neV`a}iWZ=oSAq!6~glt$2?856B?cAm8agsJ! zX#1}))$(KmnXtGiJJ*s|cFSh!`4^@FPY#Z8W>2mRnR`-$RPS2$6uZK-qs5Mu;9uXg zgc~ikemB1ve>{ykViKBQaIKd)Z8qLVBJwArnvT2AO+u0;GDJ z?0EysON*obXi;`9k5VlkD6Ir##ZlL zHdE#c$h25oChsY+;hcr0&d1cklkY+5$Cu4TkBvK0@))nyGLd3UWRs~B%4GX2v}=Xt zjVHf_H18vJ(NsQEf`5UMLc45mf#-dvG?nhL(9}+XN7<9BK`L#p z2Vefww1jmj)SS1Jgu#e~b{&T13Cn>K;>T(m7RuZdncx9xd(zU9PFZLwz|@SzHD{Fv zmOYfY4KfWDm&rRy!eF8$sywLdDW<}BM~exxq&+QmUrWAF5`|kWQS(H9dz6f|n2DBb zUyH3i*zbvho|dq$C7xaM*OKjP$+t@IJiSNw%~>?n z63?`lm6o)AlIpQTE%`_Z{!Y;nRUYa0*rcb$^tEJDEq0?M4_c4%oAIo2E%8uGve1&+ zN2{K!sU`18u?DYhF_V(xd*fiPC2DlN<`h2()M7eP&>Y4SDb$?KrBEiRKSnf@m|ZC} zW&2VnlMbX%CZ9;5Ot#Qsw^A?_hKpU6q?GmQfSK9lhtQ??@o+f6u;LYaKcLRb1SG*_Q0cbyJtv$$RHKni;BekD?9SF&QE=C`7! z`?RtdJv0>$rO;HmFNHG6f`xjfbi${L9-7KJER^{OGE)|J=6uaUnRAe7^!=VRXtGe| zJY*IuMNb;lo>5LwW`Rt=;xchZiy3H1=UQz2nPqcvf3mbK1y`EpLn*jFncb6uOcpM+ zMD=I+%{G{62{%%xCuuxeOp%z56l}NgND5`LsT9hXwG_(O;5nk1GU;3jGB#Li2^-H9 zQ zu6QJccBKm`lrilWmOa#5M{|$GUAx_5q0AA;Y*}0;ub*1>xHD(FTI`U8dJ?))mIFO_ zHIjvvsQRL^nWm1$)P%)N#cLMI>>^WtaXIBOro%#+i z3tedsSK4DK&J*W*ER=aBGFwkhg-wwd+5wFoTtfBG{^B! zi`i$PsZU|*LQl?xtXR;K$2Q8>QmjeiY{No5-@w$7uPe_}OyNmFQfSJyS!imBsR4_d ziiTR^1q)6698a zPYO+CTPf6RDsL*AsV72Dlf|9K4p}I36f!dww<}vo!7dYQwS>(#m%H2)UVGM;G+0TY zo_OaBks(Q{T2iYe8)(T_O7QbcEm8F?{;o7`X)!Y`>7Ew5(vsKTDw=WUvuwaZclB7h zt1QKx&%&*isPZ&KuqmS#4gGFG|YbWg`~KypOKyUFDR^aWR@&06Rowx!Mn>{G_{YZi6`HN ztXW)-slBJ1a+21iSiIc3=}5ud$n8i=zK}vwS##<&=Rw$)LYaIZC6DtUnoB_@3@Yzc zJxNOnWulQ3>M=7Zlu7FEQ$1Nn3S~@J3ia5Lk}%lU5?0^uH;3^+OET1w?z7P2bs-*| z>RGBM>uAXrEHw2i`q*GO@Cf1!Lsl%5*&uV+2g+uWe?jUjZYpZ9Q08EZ&mAl-6ZfRx zj>X|h3hr2BsvnfQu8f{Ki|a|dQm7}Hu+a7zrdBMNqKw%}pmhYq%A3w$p3r(GXsp?0|9yethQfMmeu+Y>)Fg0T-rjlUJ zLYXcym9zb3JUf(yGEYWk&fdoD5@<<#TC$mzy#8^&2S2oug6lGNCIyd$iB?jO zNrU<)R8QQJLYZi1Pi4%m6w1UcDYPq{NTH0We^T|>Kug|}LQ`4rDKSN2ds46~4M$p{ z2@5?I&&Fd>|Fr0#sidpL%vfmZ#hB`yqo%Sx3uQ*g3|NXsIkh7e%Df4gLq1dXkemsr zvJ_KUP}352SZL~ln5uua>~T|ZmxVHW$n;oTCK+i-*DN&k1xy8>Q_a~x3hul$wRxGL zsq-G*51Bqok+H!<3N_oUmfU_`?pmRz#o~IRu@v;=!IFix{}xkQ7B>|) zzfd+qXewN?(9~aPip5Qt>KD~iTw|fB|6;1n;-=C-3hgpO7Mj}0@x304o3gcY)vmP7 zLQ_}6)PTiJcB6lJou6lB6+ zEQK<5A%!wg^=p1l6ttvJChkZtB8%2Lc45N3T3j96lCIHPfJ)?sOG$*Bnc)`s3%@&Nj6fjJ&9`H z63sM~)}>G;=}4hWHjzRZQ~9>)v2`U$(A5(4q|iSz+L;vWO5%+a%H)mj_`4F*mV$ri z3!;t`>d7WjC=;)wP{!_DAexa0O`s+1X|WR}_!*;?d@Tic%-ZUA#V(S(sU#12TEdAG zoG{K8QqYq})$fUBWRjpMg)*k2C7npY-LUzdl02A8p`N_)eX$EudDPMpkEB?G?-1=$ zOp!zbDb!<@TGEvino4$mAbN16Chkk29=j)nrqZnxno8^sMGsBo11U6>1wZmKcm9c`k5%OQm7~HYDorC&>TfmDKwSNrBEiR{7mekOxBb_ z88ekanS3FIGFkQKeoquMrBEjCOF>T*?MXofFP!rW)f2a*P$t|+p&nDeNK8>C?nt3b zI+8*evoD1*cITI>Cv8c=qhph?6q?FcQYd2vzfwJRECo|>IMWhsq~L@}(D=3Jp=R5a zLYZ(Xg);d<3hhcNYtchunp(1z7TfrZKZU<{w1hJ)ap$*cD(OkVnN2*Fg7YNlzLu=| zJJCZ^X-^7Gvh%NhoG11yd%dT`Za@lQyMLPu`P4Q_(<6JePv0G_3z#HRl~E zl!+!%upMtZAcc0Bjh58@K{QjwcC_RZDcFVgFZiRFBFTnY?7k8_Z%0d1+4wy+Zc3pi z@`QaY(ViBw(2`dFB%0A<;)WJuwWLig*{&9Qf|k6aB#j1I;-QvgB*msdn9ro(%xSij zf(+iB?a%&o;iF7TZl%yv+}4tewWKpGc27&bm6GF$x=HOX;yl=eXQE1>jOj?BOgz+* z%%sp%wx`8b{;GCmZ7G&n3Nj{GN})`; z(voeoz7GKoD{Oi{*mwd6f1G?ndZ zu{)RXd&01(B^pRUPZU-!tEQ5c6v{+xDL5hCYgGzm;;j_QWR1(IX4{iOQ)z8SWJt2M z6!b(vPfIwELR0x#3QffuDagcOaCxzdGD%MgWun>@R3^1rvZfT8GE*(-S_!pvSpHQQY&=t;u1mS`dcJ@_4ss^4S6v6f_Ci`luVn4(>IQwlZPkrZT1 zxR64bd?ST2vAvq#Y?8hdOr_CGOT5uys#h01G?mq~*tQgGPs4!}n#%X3P$phTK_-hf zQYd4BL;dYp+>=6?bfhGU7g|j18lor04<qw!D8A_pyol2oh zx|Bkhd@F@AS*_-u2S4?cLK)kZf}SMoOQB3Y^aVc~981BJNf%OLJmDqTxwhC&Vs@p_ zRNj$7nQS5jnKYPc3FlI1DqkxJgT{5lcIt^+T1;0WMm1 zs3)69p-eoNLK(Y}Ld|LQ`eGMla;pUII4p&xY)=ZNY&erbnS5VKj@OVByO7D_x|XCP z1y_-nkrunslGp2^8B-=~YKdD?Y>Kb_Mp9@h*_T2YyHyef^&5&^$b?}>OFYqH_M~7J zUcKf0AoV6NfuD7R~6%@}?5}=7JQOO2%5uQcJqgVrz$sUFgBT z=xPajQg9xdPo(7d)l##OLOrp)iD<@D9(AP9R5p=9Q)VrNrsBp;MGsBcrWBe=XIiqY z6ig*y{bqho5_Pqh)3sz{E%pp0CYWdm*HUmolh+#lb`u{b1(`H#Nx_*-a*7nn#C<80 zN#{~1lkD7F?aFqgP$un5K~EZtq)^5#rBEiUA0e74lXs<1CK+ilOD(o?3(-SU`L2>I z=t;pY8&0&uOD(2yq-dt8w5i1ol;pu!OSqJR6WXkDOVJF;gPE3at0f6;rKU_@3tn|o zWNexQ=~4>q%B#0lnLJQ}-_Fw#4z)xxE%8!|sRaJ6Fso^?Z6#4K&=SqG#2YP1GxT>w zrmH0#DT(c#k|fx$(6fCG!?S(rk>8A;uV{$}TFkzdY{f#&H>KuSHQNRYWddYcp4+f)^C)~| zra_;Dny1LjSm;?*NowK+3uQinOx={{aXnF!g)-+L(_?WxX2e37^N?AxxJ+72RdX_6 zp{WI?<}7ZzS+G#%hse~jvWMg%h-GoR@{Sa`8`+eFrv5;?Sc<7M*s@S&i_GD++~v-L z=arX2nS*YFyTMXqvS1_y_c@Kv)?)Uwq~~a{b1nH!UT!bWjIY|X;Ojw^!RxOoF+(Z1 zKlq&yDL8XxtG7{8d0Pri*|C;p{b;Klxoh# zTI^g2ersNe#rOPa^LBm@-sjpE{8sp$6q?HCO7Qy*M|(Z^cD=48=}JLQ7!9=~`&vx( z_GL4D)Ez<}b)H-kGWVnoxyX|vARA8-NV6%MkA}28*@bjGIUdsWWbs!B-4u`ZoxfNuk#>q*BfH1$~8#exjpRVc|9ER;C~ znc&XlF4toRER=aJGIN%q$Ar~m)l}Y=f?X!+Nx`2%{PrRXH4o9ea~HoS4Vzk`HVaL? z22(v2cOEmBLc8Lf-Ljdc&Y&x0aZ~9~3QZ+@TFg>Qx@Do}ccZy}*K(I@&QE5c%#6;= zQe@I_PYP|%=TdMF)2MPc(Tq$M>`0+ZQrBX-QfMk3Ye|+;s3*Hfi(N~ho@^rpQ&|w) zU7gVOrBEiENkNYdR#GUFY@|>ovd4*L%9y?s%EWUil(F@DsHwcmLXX9#@K_8yITtcx zDIOggEwse7dzQ`A^9@W{PnM9bCqIV_J-HaN=gHq8Yfmai;j6EE$z4}~v^+T!GV2 zNy34aXraY4Pw;z^bf_gin}yDNHn#71@>$55r8tiXckb&qo9GZNNmYw!v(T=6G!I!^ zbGFAqnQtRAXK@+3WueT4$W-pV1Ap;il;Lo5ndrI=~d@a!hTH+-OHQx=*E0&@;PpS`++fP8I&f+q7 zmEAJqByIYVMNvyjJl2xVv}9W)aZo+c+lAkvZL`o-JOn4~dD4ZP!gAn*_?_7S3r#&4 znR8iO#w=JU^K4`;^yI~mwI?IU#*;Te4tuaTGoDR?CwWN0vj$^3V4+=SVrt)$57I6c zcfx$bLQ{Ll9Q+WqE2^?k<_pNQS4>Yq z-AJNCS!n8}$kbR|b8J~C6Cl%MaT(Lml1{bQ3$)}*B_>>Hi5d^{&umObOFCnr6W#_V z+-@l?a2=zl}E{4 z7eQK{`~lMOWD7aVlY?%LU7j2QsXkgZUlY>wqz)N*as*_>LXQq!TrjM3Wlw@ko5f|~ zo|a_JLXXbTwEZ!%XP35nay(?l;+nG!3-z3cjD2j`<1)6-LYYS)GxekgS$pzKNd0ks z5B|Pjp`KHb>3H%=$c&}f9>^5*ZO{Qlr*)Np%%N)lGmQ$_ry_COFYz)OtoYyEw=GYzc~pzTB4~I zv(S>(o~3#+D+PDMq#Z5UR0{5)3FcbDoo9F&gUUcj*wT{Bw3w|HTYbLj z$=gya{#hm&XfgX*>`F`Cc!6j}PZTw^#6vCVL`$|-5(kwR`pt2;t0n1ZF;gjcEt|;B zrQn3IU1-T`r;1&e!c&5zU@D24TH>LWbfP6&v(Vq>U&Y_%l^2z}NG^c1JoyP^;K>@Y z@Z`^s+KXk+B@pY$Wt;fE02>a*e+S_|yn$;IyS$8nOjr(lbrIS4To#(TJu(Xxmq{*W zq0C*8sl22-4>Gu8dD3E`%zcm${`S-I=L z_7ESMEY$oWG+n$%;S)$z1rWEhYc;!u9mo~B^k0%^Hb40V{zL}W;Vy7B>~Iq|j8jWud7nVru90YFDz$ zLYZqI(`G?4^&|r=W}k(oZh)zkCpU)#ZxFkv$M#rgDn@3H#qG*hER?w&GL>=JLvj~L zi=~*d(MU_YWTB~hV`}5cgCO-cmd&mw?Xl3*Bazu-aT&Y@vY5h0iwzEWQzz3 zM$lVj&&SYH_vCYQW)|0+oW(-hzkTpLnyVWE3?Uu1%JmOUUkHYMl8tSOUgLD zkcBcILT1h4GUo92l+Bd+Br**am&sZzl=&j*T`VpWuUIJaJ7ntb^>-ynSBn|5(A3{BHDPhh***(p{)f!Mlgl52UGG!7 z@{SVxP6P|}Tpd#jmSUF)FJYm~^^mE)-)}bYi7b@488RIf*JGwElnIepv$#xJJ4-cZ zyDT(yTWa>?&X5_4>&gGhLR0rbrt$&5ISs9rsLw)E51?k2q8YzW^g+>s-*yP|8VgN5 z0#i*EOd%5mr%A!bd}>Z-p{d7XYQo}nW&13Yc{(!H4~gy6V>>LAc|I~@7T1$6q|kZH zT1(oPmCe-ia`bdqik>VSvQXxA$c$KACSFOQU3vY(viWV)%;Kh!i4;uXT|A`VN;6aa zi0pYUdKxU~p{b}Xg{HDoSm?|j#ngbsO(hE!%6tZy>PO36?u2X zJDG)g?uty0#bxq23uW$uOy!ehb8*5jIz|dkm|RTh^q$7xAVV4OZ3ngJt z{iff8Zv(U>O)X}iCEM4MFO}fg2n&B#6!x?vb1kOwEj5+Zwd8@4IOu5!=US5B+kSIw zdRnrPmVBZl307L7#&`T?yeftkGtiPQwB-E@{2mkRX^ECvlC>5Sd{^~keJ%M=37*HO zC9HnW?@5!67BkV3&9&qUC0S7YzTcdMZ7DXx>xy)x;JFc5GL}M_XeosOvo zSSa&3Waca`lkWUjG~+7pS{71hSJq*np0A*%$5LEH8jYmTlwD}aS1i=?9rRRwqP8b> zDd>rUT`6=z)0RS+aKu8*KSuL}#hoynN};J}ttH-SNvc0pC(IjCsK<1r;DlMwmqM9z zswLZFp)>!Q9vznA(TVI*3N^LxM885f1_vT zXR`S+yLk2pOL0Q{{D+06u7XUT#cj{VQfPa;WTB~RV`}ZmjUlz4m)l*BwNj`jX=yPX z7HYmFork69vBBw5s3#q3$!09nlc8tH;(DUWFT@F{$2O(l@xnXZOQB5OW1;3dpn1T8 zW@M6JswJGW(A3@OJS^@!Y2_mSJa|5m6gp4VmO{JY0Sh(X7tLc9cb;S_g?jS26zVY> zE$PlL#d$E5hjl45m33HX`$Oq&uoTyoM++%5Woy4Gn`!DXmzgm^GwOChoA%37>)!_E?G&nqyYMP}ER^{qorlFWo0Swa<8Q0qmM3(XG>}4>XqSbWzli1zi`yR0SSa&# z+RoxK$wmr27J21&Vms|JO&03;K6=_L#RR%HelK?6+QV$XLYd3o6`wa)Tu;7Wq0H5hss6$5 z!MmJD!QIHxFCqK?KkvOBrg}PK=PcBGGh}xDsG8$lDb$<~rQj-}WXeK4A$s;%imQmy zi&-dhTVyt#+!@l`l-ntDFG!EY_1Gy3WgdXcn#E=Eoj=K*M<7%8nKExfro-YgcEUoL_ad`q zDKdD?;lIkBk0KMWxJ)!)p`OnmGiPy`xbipI^JQc#3o`T;-%*=|dcK8B$CDpHW-M+h z*=RBL?{e2KG1X>qQ`vxpcKs2VkthFztXSMsUjK(|zVzMjeG^NO34<{UHD3vtJx{I$ zS$c9qNM&0#yPoJs7V5bLGJz)sGGK92$y|%s`DfWoJ-5eHgT+l{Z5GPh6`77F_kqk< z+*DrqSJ_+$zAs>*o(E&9%i=Q8h=nqbMrPv4lOP)wHhlg*dAJ3b$Ja#hIGlj}kjp4~T}cE(=ZF1DO_!%cKJ-cz>WEJXZ=nlbCG5LOu6K&xQp( zG!-3nP`RBl4?|{`#qG+6ER=a1G9wn(WA>zAJKk^N(z5xf)Xd_h(!La$iY6?y>v@=( zd-77qipBM0M;7~0E}+g&E> zvrx~6(KBHwwujM%g)*N;rgmA;L(Oqp3T@B3QYe$0!a_Y?LeGGuI1gUmMoYA1p{Z|T zs(v}OJ?pVh<_E~kSlm_EjTD?POKLl+CvUJ&&o9tpS&E)0I9UqyqysJ4J`45y9zB)I zm%B**0co+g=6EcHn)9U;+`}kYvrx~aj>DhIE0oQy$0SnFW5cEt>Pb(PLYZj9Ld{o1 z^W2kb(5@@0UHL8xP2B*QAxm+dIG9PHUFk{+&J%~VL&|3AxjA|op2Uz2i))U~Vxg(q zA+zVnT_8(O?hUD3N$z?OWS7M?o4%HGkA-$U5>p#bo(O4PSvEfdGVtVukeMg1fK;v` zr``bB_2li4o+s~z%su%SWb4W2AoZ$j{tESY@*T*`lOID?p8OgTTvhh`3DWW8-;jwX zm$?T{=*d+e8&9qcsb5Vt-x$*Kb3mzT3>TVP+wL}-O(4&*lE>G?NXnFjYG>G%G{TlSzN}BSSa&QWF{;{2Cw(dLYc=P)3`?2>@s1Sg)&b; zX2jw$@rs2q&p~Fx;xb8aP0@@$arnI!7Mgkqrdlj+DjjLbmMk>&YD`tHRW_5n8PZ{K zJ^7S{GVelWkEO`q9m-fJ^C4umo_rEgtChPjg{!b;mxZRjh)j#cWzvBZeEcNQh=r!U zj;R%ko63S~i(NDow^(TE`%Vd@7 zl~a_t88THCWbk{)c$Z`;xX;E;SZFH5)QqKQPJ_zz%VyVOT2j!HhGQwTD_uyTOuUvt z8M~E2ndFcgl)LCWx5asCEXDRPXi1@|bizVYcgED3#Z84f^|IMrR~E3))V(m(WpPvS zSPJdR=Ss5Rd?~oD(5$qi!43WGSv1fR?`tuY8>y-E5G}T@CGRS+@k9zvh<~Y35(l*# zi(SazclNc!LoH^Xh3?@4a1U3WJOWZZTs3C_3r#&9nLbN#=kX3NQeumjY+PswS6bqY zmc-tq+)lfmj$J*LVi$gkUkdF?D>s!>&!?+kIq>m=H_quw!5uSEUkdf)6BgS3a`bFi z-1ek-vvOC_jQ40}p{dtlYR=-O%u)(=g;A|h_R!SZFx6mjQ<;@QQ*n!hrrwLG4vU-0 zXHsa&RBkSJeH2r>EX7nB^rX;KI%c6=pTX3E1yhuNe3pM`; znTaRAq+Kk<6kb{A7UeD{QHzCo{zyHZ{1dYA`M=X@N1u`oZWUw7SFT7RR!M{#Q!K0JsEno0@J6#sq{wg#NSPnGfXG9AY%DfSojVJGf z)DyKm38dg79$#IsP|pW2HT2{YkT2tZ;cKU>FcUBHdD_n6t`x5#S#GE1uOV~2ONkyl zUM6mNneQUgVR1cnDut$!v!&p@v4dbw3T4s_3vK@?nwx2Po&(Kz|F4l0n#z_eH1!)y z?PO{yYDl4}yd{OslXRpYlLjZVQ1f5V-1Fo=kU5JxVY-q+JyG4N^JG0Kl!^DG(0S}y z37*H8`#oV)*AjQMBvUQvLW^BX!FjTzaT~QOZ%Lty8B4*L@oGp?D3h(U*xU z(_wKLJC%Yzf_UW|DRkz%dRN(V1N78AxjCfAQZ$=zB!!xjl@v6asByQl8B>%o9VwJa z`cf#94W&@VOr+qm2A<(Bg{Ja_6q?HFclWo4L01Z8>_`eVr~6v6l@`0zl2?xtyU>Go z>eLc8wM1<#abHU^)?)Uwq$@31V#=m3iTu-DKr%= zrBHKLZHXS5GM0t@4957A(q+NNJWbi26lzY!EHrgHOr6Q%rt+B-+Lf)OP>-qJyF3r| z+=c$+vJ^dW&|sm=y^(3LxJ=lULc8*j6xx-oSg7Yg=xH2ZZg)LVQwn;LpeKcT?3{&q z9*LgHeN<0eW1-9wk=bQ&yOORHH0QyPg{Gc?sTqr#G7Bj*6>eB)>V=r9o}kW?Hne2B zQs_!ePYN~1a~5iT1)3izKZoOEBepyD6}xCE30P?A4VW6SxU0w~N|InMg?iF83-!Dm zJ(c^3UD%$4Eh#jW^;u}@{g@iE6x)+%CI#1Sf~6E{j;r?<&e9U@NujA^A%&*mtrW_n^@sV}P12P@nQSP9deVgy%H*|& ztDdZ-#m=N)DhwA=s5!4aLiA833Zx*Dh663pL`%{dA_d#=s%4LnQ;)+`-IJ$6EQ>o& z(qW-p&qHR);xcB=LYbE$v+?A$kmh5{?QSX?ve499keRW#jNNCU%zKboc=BP$l7*)5 zmsPN0p{Y+JbNJ)rc4WdpOEh4isV`yb(fqdz$u}W$FY^P)g2io5>nE4HXzCZpv{{hB zBbWvQ7Rvk{nI%in69yX=%KQVF=Hts|*AtIeD08X%pohg}lFAdxDau?CnHTYOxlCH~ zGS@(+$>K7$#X>zdKxV??GWkXd9t*sPVyD;C-nW2)XOd)yR$ zHChO!qHw@MQ@6v^g2hcG^(U#RsK-K6cfr)YC-;WDnC}>69t7EVnMXpZPcEA&^F&C; zlV?E2p1cq;^W+tf%2Q;|8z3+7w!a->IfLuMwc~ASSZMqEso9f{QS&Kfb1{_!O%|H^ z95UzdE|-Z~Ugj&vj9LDVv9p1%a(o~7nf0=ynkb{UrD#+-@8>y`mc&R5(Ih!VDT+mD zDU6CyX;B&#OT{WLLupZr!Vu=Q<5(q%pc4b<6>atbK&q4jqu{gUK-f2pmj`26BbNXvRm2$;tK$UDDEMin zTX^S681;{u-5F&SY0h{oE8;luN=~pGt5yWJ>n0eLRKy+yU$~j>&JF}e#d8(GQMbaV z5=Cq$=C9!d_C_$2KEv(csJmg*zs6Ps!+Dys0Gwh)ob^PjIf3=W!oFL1c?pcl)yOI! zrHVKmEB!WiltbbPMR0}Bz^H?Kg*ca0ra2qIsa3@8go2s3yJhTb0ViJ(#|c#`g3i0( zBsH=fNcb@KyMgF@1;k&-%X@(2YGfafQjHu0645#>E`e*4<{Ski^A29_1SDS(d(Gi! zwIaBl9^j5g)oc%~xg1hAus?ZE3&6hY@SaB38BoM75aRu0ACJD3@q;1}%SZweSr zu#6LQB2}DVC$!5Xkvfw^tyxCdDlkdB#2|q{xk-Y)*?K1s&M`?OpA%Sf)GFiz)*Ogc zn#4-KiDdtp`Vi{fgkIhhS8JMpghR zQ^e_5)r#P#r@+aW%gVSF@k~X~*#M5Ekyn9aYvdguQSFiB6CgR7^CggCjeHLzsgd7+ z#I(*oK(gn#E5!9Qztra|)X0%Qsx{I9i2pu5>I5L!8tDV1SR-*DQ?%I!0;$xTAwYcd zd3hL+T#bwZa|!nqH)qZ}tP&LpuyPVl#DMXETl zEcgb3Zz1cT6U#M;RmusBf}g-z#6}?r=5PX?fIq z!3oS243=?%PNb3(bi%$zSQ(vIhDod}PH{h6V z6LbP4oM1Uz&Ivl9N>0#;CQTB{DrIxws7Ss^q7|HACthcexZk(j=)}`Gfl={bf)n_E z_-!PdpcAU$1f5XkW2}r$G@lc6!WEpL6RYL~ooKBh@Mz%s2_6B`SGZ*)e*no;gzoQQ zzdx4~>{z9W;HZ>g@N8KTXH+0#rCSDqqvBbL;HZ``DxruoDp;%tI_NZ8aWBbHbtCrB*_V^2mS)jDz-xSZlyn3zzI6xB1Len5R{)(kIzmyR;D@qz^PKi zS#!+yxH}hi&ITt}5xW!cmvVx$TeXVds0(3~?+Lf#j0)tKBsi24=tTVqMX+-@be>XM zfn*GjBIVd?js`0^f&CMRq?fVTu^h@%1UolCr&tkZh0&xU=u88rP7!CWNcL*CjLuAO z@)U8LSiVWD3Po_#JQ$VK$YLP=HN3nW$kW=kuLY9Oob^D8HS!9Oq(yU3Kg|7WbEHPB-j4#XG%$ zWGmu~3Kc7Ydo>15QW3`qXFkn41HiEqahzC=BG?%WPOc)3V=Yw#olC$eQ-mD&J6TCZ z(76(vefpNd-^t2)hOcKlI0;3d6Newxi58k9R;37*Z-P-tMV#5gy~=s{R&Xpu949iF z6PPRFFH{7}cf+Xte9hIc^P_9YB`6h`Jnj~6kl2`>N@Jdklxg?W>k|qgfy}(uoufGX|vN^#{ zEZ-zn1t&Nvnlwo)bAwT~EJg7Ac^A%~Z1|Ubr6LjxJ)($hTt_rv5d{`Tn@2@8U#KcR0P-j4s^;Du}1|0 zzL(t&I-h`(P{iIF(MS;|FgtvPXp-;&MX>WFbkbgN%TCA21SfDlN8@FjpcAW9 z1k2w;IkQqP$Nc$rpam5MlX zMbckm9UK+%D}tk%4~L_y2#i7}Qq2kMjhL0O#pp!y6v57s&>5$QGkd&H5p+6$Q>lnE zmzBQNEu(V+I71b2oOr?@p+JpEg6Xfj9eWh~y;_q*3QZENGfAx8B$n?DR))P152kSf zdkns-s0eOFAJ~dwMeOy2gVl_;*Fhdb^uERAeVy`(8EHz2E+9Z+8cZ_nZz$8|oL8AUjlLXSY>78h3vPr^)CW$6Z z67#=nbmEl;iTSHd5=?(j9~BEFID!2fjMi`h`!E(sd*2upO;-f>>J-?k`HI-bD;Td* z1fAQ#Nv~!dmF{R0N#|z^PNjUSTMl z_MzKxMg?;?!Pz5)ieP6MbV?L)Mup2w67hZHmT}Y*Fe+aWXLhSn5p;2%5mbE zoWQojE2cPs6^8u@PSCN66~XdLP_9P7!(cqu2?iLK)VoS46Y6LhSkL4yA5 z&-HRFRKy943I?h1o?1_}A=6v0vLU{v8(ZpR)4zh6`lbh?0(@wMTE{EDD+5;*mWIGsq7 zZ;VmVbVYDf2xr$wKOohLI325w6WE9GfVImg$MY4z^4ZWy->p00zz9Xqxe%O$A}|-0 z!^NDyF@w8H3xdh_fDe!Kmvv zqhb>j!SW3-Dqj(B;!$|1cOfV6Qzd?X6(_htE6E9t3gvw7&V}V^SXRWI9bRRj2s$&t zN&CSa<;)(jIDxq$frKJBY95U8?J-8h(mBCVp*&7-g;s(Sbi$>IV0kf=(|>f!NR|Uh zDB^5IwAduEq(R`-&Of;w7zO(<5Un&xyv`u-BXd8qQLw_OmCFfqf`JlFz=`?GIYB2} z$q70^-!H6;P9%pDbV3PE(6Ne560hbY9))k4rPZ-Al2AG)P`3P;oS+lU=LDT#1t;jl zsyIO>oa6+Zc=x?*c61_sPQVET5+(_jaROT!4pf^Y?E6(O$Aj6N;HXFuCop?FT)_!C zu_{i`i6%KgC!Y12F;{FHC*VZz zsNX771fBKZR4YRFO86NB|30G}FW>}c50`NQYmSDh6v6Tn>&*CbX6 zCs>YFDuU&YpyT_4bWZbjmgI9hU1^89TvDPGBzhnHo;8 zY?X0>qe2y&;HW?yCon4L_xt+hFAoEftC3Ma{@`Ed$Msm{nsXgE^@`Z*f!}BKH!n{C$FB%Du%%Y0P!U|u?ckIv z;&g((18&D5p&UhU)IBh&9sEiY^;=D_94*kC2f(RN#F;%-qX>4Efm5f54M6;<#_U#sN#aF{U}qYP>Y%M>CXjOF z*z1V}k|qiIo49jfXC90y*2rQYNsTNAlGW55<&>j-MX<9LoI;JP2U4vFjDnvi4h4M8 z+zyU<1)Ow^yagmv5qmBxl&=Vm`UsqIMVxYMCnxZzWkqUD67w~8XLrh04kxg}aJaxE z(Q=bmwI+%C)AVxOD$)r)e>O=lX_8P@3#}9LNAgS(Ei{Q$YLa-JK?2d-!`(9O!_Q$K zmMTJf1AZ4=y&~v*hexNSJIZ!~{z8)ksuaOddtsFC2zL~czk%dxr0Gcbnv5b&IZ~hq zj%ou=rACeglHSTKJEP)$MQ~I%a0)fj8%VWAVnBSY-42c#03=rtyBvy^D}v5oa5}1c zoa0#ansW&_8ExFM?ZDT%6~WGx;1nz3IDw=h=!^#^Gu`bVxd})oH9L}9fsE6fyRod1 z1wfLDICEM4BYAlVI0cH>4t)1t5u9rkmKAZFU{Vouo&hJTty^}SP@W>_Yy_uBBU^x! zYUEuYb&A-Xs6X>4y&MfDOcKsFNu)v%oP9f%HS!gZ?4$K^ELd!kaE&6^*@GR8?88nw zqZ3V-Bvz&fb`HX*8jZBL63QAm3P^T)y=(>YO%kazN!)jgI|}FO1f5)s^Z-(-2<>P1 z%TT@yJ}Q8tG;$h_Qp6b*u2cl)8U#+-v2Mq4BAJSya{)NnihvV`Uve2J<|Gt{A1|*_ z1V>#4qkPA)GL8!RIl)o!T!X+X(>TFVkvfw^tqyJ(=ei2!N@!#vka9)r?Ft4nJMvLe zag-vC6Dd~&%XffNuZZJB(>w9b9B}-KIF2<`5$r6&xfHP-_>J_6pz|osrHJDMtj_LS zKwxiJp>ZY&=bI!}$O-J{aHyIS>{uCH+%nF-8n;3bd-gE=_F&hsXNTYH!3maQR;J+u z^Ay4I^H45U#3{#jDuT{taB3BC<_aI+1m=na(!08I;ixxZRGuQvsAwT4I4W4h32b{j zR<8(lKEQ21o|pdzq);QffmCYb7a&QE{E3}zZim(buk_A0Nwm-;u_}WE{YjGqvbwuv zTytsywp}BwfRt+F7$9{T=?WzK1m5Y19gReQIsVsI=) z9LLI4#0R%4z)2`#JK;dNBIsNLPQ4Ws+#ZB(X9@usjLM_zTAlecrth zNYZiOQ){^G08*=wKY`R~i{MR)|n(^oy3j>j*5&k zNp!LzuuFW?U~d#C0(%2TS;dN=GZUN|MI0wyuLwHxz{xt6D(5%oweYUE8;leq)8%KJ>A*CfqfXa@;HI*3df2}VlCtZexZv$P->E3k`w4e z{Jvh=T<~*#qNN|GXNHHhqgi1^jUdaiLij|whs^bJZ(e&PIJxF4CoInSDUpFV{ z#7h;y{k$Icvm&$);kS`gass2G{sWv~Ih1yaK6^BnZIV!tNg@@B;Owu!>{W``M>!hZ zsR%l6fm5rIkAV347_(cMir}cv!5OEBQ;ruZg3fo~lq+I8u|Ty+f;%~ZZ4X9kIf4BH zKVRwB*At86aDt<(0+YlmIl)fM7ce^35u89L6wKfRMp^y@C+LLAI6)^+VUkd-Ny5LI zB#<@vCW#jsB<`PUl3*z(u)=VpmJ?WEJd_b)bDAtIj0wRLY$l&yBrsZsaRs-!CG;GEU&9W&DA1PGENUTeO^@6Fi^@mj8xwdc>W} zDM#`YL8s|xm|YQYuoEseNu)**9MuL!rAJvAN5!%fLFZU-vNh5TNRcAWT=8;6a8z$_ zepkeH0{(iFz|Z-)v*V~3jLOx>03aobIGu30Nh0+ofnV^>%Y(73kxQ_wh*ORy4HEQc z#odnW1cM1haLrdjr%)r~fmA8tj0&fpYIGubCW#d*g5{f_Q?7{9vFa5;=T>miPt(gG zf38X3w>=w9sLmwe^uC4@%`{0Y-z4!$gM|G_MR0|8!wTyav9|*Lw0u8(F8Ev6ir}aP zFe<5$B|x%HH%3LWO%hEgf}K?`s!$PUcB@nobe;jHP7%k6XZLsKLT4j5rHX)qzpyP> zWfH5#B=Fmw-Hts9e#dyWK>~i01oKQ1Dl|#B)FhEAlSGpS2?l=T1n%##V4X=q2TT(2 z4Pa}=QPCWeSOq4D&oYP=DCPwAVJKX~36!m1+8JyvbRrff=!Ek)K_@nj6LcbzIe{x~ zC{V}=j*3@tf}>*T1KI32DwxL!j0*b;IKfd?i6VHmY=N_-N)b5j*a`d2WM%9GGdY1y z+@EcdKrScPi6%I~PPj}Foc&!qpA~Utk6D9^*(3R!;OwDtMX<9SI`xV`2c2;GS#AfN zufWMs#Brhtlf?2B!BKl)RG~)p0jW~N>BQ?4!BGdn37_rGP7eHr5X&UNd_{0ni>qKR zMVwKgDn-ya3Y;2694As|lBkvA&W@ux!Ki$V^Z-(!kpPftjhqG~;~cjGbHNdWm%%B5 z>QD{W9L!a3Pqew#Lo$i3gs$-qgKPHgd)zUSQ#fcDpIWoj(Q&F zy2PE`*{*1|BIs-eXM!S5Ii7Ejpue0GoXgs2lK3t~u>2;J>ooEK5bIKXJu&!g&jtyW zm?XSa5iI`?I@OB6n&G$y;ztiNI+2cw;Hcd&DpL`ARM4Nt32av^mTwZPf)ki47^+hQ z%fCQ5JCDtdPPm8@C|mxOir}a}VN|&y_R+BdwVYrlTE_`?;P*cpb6E*a&cakf34HQeo>(+Zq~BF=iEC7i%xLMUF!3C0g7u2nM-YB1&XsH~SWnEKPy|O^3#0NiG8ssPB6cShs#XL?-3Cs2 z!YIR!$1noh6%6Jnf}@IHRIws-{=jd=Nh*TQ{orJcGCGljBIqmyXPqKWCswHlI*)^s zIojyN^A$m-9GnV8>`usEZIWQ(Dz}59UW8GVia4Xf=~r`SD>!+II1c>QXzsj+D;&c~ z4Uln)z$iFg@oddCXp znj}=J2o5?NMrB`XjEWR1g3i(4)F|T29!~`y*Y)6RQp9n>m7KsY$cMkU$4S^9g4dbf z$mc46PQD^eCswHl&UGg^{+rxU;NY*`_ve@-P{aw&6)!bNz+b}&c0%=>z^FjbH`$mg zlEVo)p<+(36HS|9jEZG)f}`Mf7aNXMzzI6hVotCeFEdEYU&#rMvXY#@s8}HDW;Pc( z@mx;O309gUl;i|P!QW$@s(0W|;BbPSXf7u>DqPG7Iai7GqQ_j}vqv`6h{0a)K+2XHM5Uaesmn z>{t~hiTh@-QQ(9Fg(eABnxXROiB@ZJ@A`O?r~>Fr!_c*ia6zXH78h(WZvtJ zvK{z)Ns3@61Lsl%9QeZ{;ka)O8-<-%Iw!F0;h@C{I##wKSUw)g3ZlYoDlz<-CqfAEtpK*lM@85JrsNw`80 z9Q6{6s#3%m6{$8!G^q%VdL2g9DdLQZS;c%S-UlZ~5y!C-CW+_kBS5T4a(~sY$FVMR1SpfW1+pk#B(P(#TJ^ zR~50h0$#0gzq?)7`2(D6jig)y&qp=V5=cT3yJPwD6~RtBa4HpXoIo`v@TY#l!CI3< z{13S6fl=s03ru3oRRrhi0&|rpVvmaWt4$KfSm>5<)JZTZTM=3zd=0ix5p+Vh6^c0J zP>mw!^aH0}5yy#SE^_B`%AtHkaMal_YLg<)sAv@@VfY(n;j{-?2md%yERz#-BH4;y z`9dh?E8>)`a!znmv|14ybvbT@BF?CI`eHUaj*9s?f%QcFd7PkQ6)1w`F;Fhm$PGX$ z6|v`vhI}RZT!BEYBG{RR9Yvf`;Up(8dmvb6l1TbPMmd^L1j{p_T%wVAK&mvd7)Y%~ zmIKLpST6_T2}Q8H7MxN=V9l_?P_SGPbk>98TcUSDp=?Fac?Fz&MVwB!$Rx2!lUV6X z-7UMBcZy1%Oh&{>*j8g=i7&s*w833eQ5oc7m(j<}e<$SKeIG08)0WwYzr(+c> zg0o)U&%V?L~|6uHQx#=EKr2DD+;eWRRo>8aoZKKoq#`cm0Lz<0XX@JI8LBQ z5ppZrK?XPjCYJ zFczL`lE^|%V6TRPr6vj0ae^Hyy^PI;B%WswD=^t4p+b{{s|*s4|E>t`=k2gJe5?6f z@K>aR*@~d^6*xtTz_!EdNByBHMbOy;POT!26Y;Ha%joO_r=uc{6Z0#A&OvYr6mcA@ zLJ@RYjE84d8aWC`+FExm7zO(oE|!Yms7~M%DB?J=VnxvD0S+vHJr6GcQllJalvSq) zj=B?^tS61x<2j0;GZ&m9jXVgXOe3X0suZ#3f>&pmB$WM>I~SJMz^HK=S%+msoKCdT zB(e0T-Hzi}*@|HKW#|-ZqzXv2BK9cwI+96({%71Wc0PnrIU3msq(~8`6RuPQM|}%U zl_HK4O`0U;FXwaB!Khq~`~{>~5xW!j*O?@c{jA$@oM665B9$hIraxzlviv%M7vN-@ zBv7mfZfTQi;o78;)<7yXk^!VjBgX@&(MT^K^%{u+@jvgb5a-GUlCP0-ffQ+ED3DT( zj0BR@$T%Rrb-a8dkW7ut0FuziT|i1TQVgU@*HK(aORG>`&~Q~)W~$ZJ5V zHL?vzokl(dlJx?gD+wf5BR>EsR)o$C_}R%SMR2eF4o;0m{smH}k+kdJSZr|TvO9rL zu1UfPlSJ}O5-m1Ktjr`PGOB3Ql|Mx||b*G#j+pBK$i z1f4=~ax`);kc1-6T(NvbaMVI@Dim=XtC|yBPquRWsN)pBwLYFCXXqY+@3x$L^XY^g$L=&I=OlQW0{^WaQs%e6YJAGoV@6N8raSz#Rk+u|f2k?8 z@D`5&-kQ_=d_66IrlvIBIh>pW@Ye0L5S^CfWWZazv^1Xj?|nUQ(x-99s5|cCFgBeb02tb$t8hpa?MAP!`Iwatodki_?o5DjvT&b>9p4z zTr+pF}ZuOD9AQ-_OzsYYwiNJNB9* zkJT`fgU<%iq1gM8AMK= zGuK~j?wKjipR>rRhPQZ*;cbxUoDDQJrSbD8ha7&~&la6?Jao<#opVHIu!qihqBB@@ zhIr_lPfjJgbpbl_J3{%2<-Iea}=hz=a)|Li2l;m7?-(HZ5TgZJXp6y6yn zI#+q2S#KM$`Fo$($z@MT0}|BM%%>pXNOh|YDQGm)Iz z@GbmzXS-F!^8S83Ice~TmHYTSQFP#GF08r4>CBve`drz`CnukEZV;VGT4&c$%i+n7 z`S^1qIYU`zlIYyzp)*-@ZW5g-)=-ld|LuXe?nC+iiMdx-8okDVW=XTM#!$apz(YZr(;Ask+&y64V zS)wyjbY^?#+$B1*Mdxl09a^onpYIl(d#Hm)77mB6S9P<_@_fFR9Nf?Db>|+@nL`eK zRRKHAeea$rJMa(y`*16~bvtuJXC68A5J`jguFahOw(Q(TP7}BTro(%9o5wnMuYs@0 z!GDK7U!Kw8^uxJhe+~ir{ix#kDRnE`XTkD2$gj=koEYpa&qvqc)UII`>F7Bz?|zb z_=x^>+U0C6yR(8E{Okah)yXsh|Vf<8eh-jsc#0PmzOH zXWXu`F@50^MDut(O-=^euBSxj8FCt5PdPaU;H|r!XGG^&51r@8NwUtfqVv3m4j#?a zlwGXzyy&d=(0PHJB-7vk5+;fB)ho(b-JSOn8fPjlOlko3isVIR&uc?zLbu>lDC8;q|BF zxQ{LE2W@;+I+f(~fVa30|8BRuEqCn4jaSLp0dEh7|Gppdcmvnd*T^Y_)BFf{4{xuE z^=tup=L?%3M}yA{@h z&pPmV!XI5{!WS6K`)L(9d2BzwDYok^a@@IaKXb>P3w9aI6@j<7U5Qq6zm;>nLrxOj zy64#2Y%aUAjhtiQE#BTZd;Qt6^DcZOH3iQ!x3f)j-Xo{+^}J8cF4lQZbnubT-Y$5@ zH~F5g;qq(l!w<;W32)u?REy4s_E5kq3C=}jsq63z*C8oOjvt=SwH4IkN_T)kHd9Y0fpR|J=tN z`*?j#4sIQ;dDxxfkC(^m8*)nEEgng{ea()S-Pr~7(D$+j?7s6V+1X9bN_gvbc8Lz$ zy`ck(p!NLzV;($VM4LZ9{EnPi@Yd~oD>}90;Qcfe{+s{mnpLv%Jvm$9Z4>wpZ?&TH z15o?94ep=SN4AG2)aH8jkb`He+xbCs@JgDR@&rWi9y29=U8d~(M9wyF@NpGyKeCQ} zzxWwwYKnW^skt<@rF4GLoUPB@n}}si4!>S-$DV6HITpOdqtx)@*dOFnI1c>d z+t22*JN4w8;y5)oe*hOW^SJ*>&PvCDe|+_#^A|bpw!b;{WOxE=cK#-(430ZIY|`y7 z*0Hz!06Bcyxnr*xJ`I3<*!Z^pOAg=mf7o1h=O8(aZ#&*SQ&af19~7MwavInh@ z)bMd{K~5FCb&q=*>)6NraB}#zbH`qDOL7`N?njWrx4k8s%kH!yr}1rXO%C7oR-)5} zoW_rPIyrpX+pvzk?MITsx1Bron%k1o__iNK&L%kC?s0F+=CV6SlhgRNwB+?YG|nvW%?@ohhj9KP+xvbpR|2XY$U_KxK6 zZSNpDoyci?+dJDGcR9>)6|VA~}59xnr-n2RV&zdlor-+k3FN?9NH#G`{U8lf$?D zB+=y$?D3oIi!lWq0s9M5!r_Z+n0o ze$M+vCrD1?+a9tz@YX%&gREn3JAQ>_tU;?F1vF&IgM|7e{vSG z&gr6)O-|$6K7gDec@}ZDPIovK@Q!ci6KBBk&F9a9$-%FvxSxZZ z%jU8>=aJJ99K3D&)_*V=RDClpPa_`;RWOrLdSi+alYu}k^^6?p>t!@ z#S!?V&0NogN4ePUyJNEt=N>1b3bqP5|Y`ccC z_1K+DJ#>bVvy^o%6&?J_;eS0oUq;Sc*2xo{;T}4dlQV;LhKtS!51lK>DP)}yqBGJ% z=Ss1jk)o3zXC%yppDp!$_LQgP`7=taCm}kcJ*?*{a`=88EjoCX{@3vuLk>UgSBuVA z51nhsnZ@?=SkW2hp);Nwe%!~2&b1yo*NM)xqBFrmXCgU#KTi;y>pgUCAg7q^pX)^j zztZww$7_;UPrm5f=%I5HIfZOJH;N8^XZgR@Gld*}ye5mz%^o^aMdxPGDe%ylMh<^G zED)VrJand$!}rfEqJytqIP~?^`j?KmLf+qRB?q4axYw^4qH`NL_-vs$c>kN<|0LPD zot%&1a?~0eyxqn+_I0O_oD<<=@Qx3^CncRb$Z@Yt_`M|Q+)0l6HT+wmi_ehGOmccU za~<6M#fRLnuRF8IY5e*%n;cxfdtIKzw#)9^<)L#oIsE>0m+0U#pZ_|a?;(djjunZ{ zy&gJq$SGvob+72m_0X9|4u9@9S9I?4(3wvT-#_<>PO*p10&@6v6^qXO9y$+@!|$i} zi_St1okir7u>HJHbRP83Su8paicX1#&O_wz?J5zShdp$bh|a^Jv(!Up89Ds%d8z0; z;-T{>IsAU{i0G7h=qx9PpFgFd^O%Ru3Uc`Q{Fvyh^w3!)Ix9uzaSxp*$l?3vanULB z&{<6me>^M`oi*g(Yct>-U;j&%9wo0AYso2vkGt2MHKOw*Ih!1(ZtK87vhx%Ys$87UY;VIXUK7{Df=_OzF0ctn)BPLJI8a!z7{-7PUCBSjvT(` zXT_SICx@?DI_tFc;F_hgUUP8G+_BgE0y)>gzQbd&y=PSepBHQ(Cl~hf;kc9G_JUY* z1v&1T@pXgJ*{H2&PxZGmr1PTY{Cvq>XK}|~^GoDh-@^VG-aV7XjF9KXCUUCb1swP} z58hr9Yu-$b`<&#<^7_N1^Rl*{J!c=aTsp64&ek>8J}#X~&FOr^#J19TRdXKBf2d44 zuW3&9r;l`$&KAwN;k55MNN20&tY4OVMmn#PMNc1H0RWBhJGfU_cdq7z8zX)f~Kj zN#{4s!RwcFe%Bnleo1E^IsE!1o&B1F*DvY(p*eW{l1{zm;Pp#7e`*e1zohe*=HT^9 zI)7^pUcaPsKy&c=C7pjX2d`h!`B!uB`X!x%nuFIb>EM@84!uSZuV2zh(Hy*fN#`)l z!RwcFQZ)y!U(#u!Ie7h&PE*an>z8zzX%1e$q|;n;@cJd4G|j>5mvmZa4qm^cbGYW< z^-DS}H3zR>(m6tN@cJd4R+@v?FX^<_9K3!>r;X;|^-DVGnuFIb=^Uv!c>R)2Tg}1i zmvoNO9K3!>=V;Br>z8!eX%1e$q|;t=@cJd4V>Ab^U((6Y9K3!>=UC0b>z8zn(;U2h zamT)Xbs*=dRQvO<3+r~lw{`((uet3=&L;NxSBJyQ*Xea4hkwo}ozCRA--{eI`RfOz z(?xU6e`v`9>11lo!v0H7kWN?4Ngp`wQRy77IdAX2p{I1ZY0mDn_PwOjU327hBDSI0X_Crfh*7kB6%kgKKxke^w*r}`_8D9PPXRkerHT;=?u`E0|lucNaqaA8GTmw zm~;ke&K;?J-;mCkn$x9UhfL`V(wtwfDqJC*vovSI6;BM3&e@tXKjW2k(#g@BBTp>3 zNIK_ePTHb>=Sk;W&3SO`j_bH%U%v*E179?xtJbj-&xMCrK-%@|JaT%$Gh6p-)Pu$K zYX~{+^$TBf%N={J^F7RUftc%jF&BRIRe!x7ckHW)9Sh--e+^&nr$za=c zp_uDpZ7%#;E_dv?hLUp}%mXQ3d}+4~IoBoRRKUmGeK?fOWp^$mXN=RisLuv?#sEk= z7Q@KNVr#xsbn-lOE+eNS>*R^faB?QW3g8{z%GZBs;OBQP7oFjvGlCp-p-JM}C%_E6L&K>PXQ^kmGLGJ!8}FlATfH_}O|AqBGh<=PI$D z(V}xTIgOt`W5`+QY&-npyIOR{dgxq3P6_Lb6`gV9bOr}D-`8tztMPLGj3)s&87`Q$Wyye5&uzrUO>IyaJoKVjMu{+lx-^<%kRH;K-TqBEJCuF$~a zwRuI}RkAaMoLTU3c!zG2MdxO6@Ui>yT<;~;x)OR>E!V3xl(a;jmw-SfFnbnYal@$H&P&PwROJ9N8KbY_v$_2gx#Tpyf98>M01EE@nJYT?k<|s3%#CnQF=Y9|Cd4QZU*e-a7Zug7MLUJ14Ka0rW=g&gXd61mO_s?Q-_<9}` zoe~f0c}T3MM06hZu%0F4@Z|{J3uuo%cL+-WQ$sM5o$A=L2&1ey$ds4?T1~B8PvT@}cN_?4k3CSkK3z z^Qni(fPqcXAe1ruto3=-F^_AA3bz_ zB8NYZ`cZU#_R#r-9KL^k7M(f|oxS7~!WO|hbgL7cUp;hwBZuEFeifbHJ#_YwQ^MBs zyXfrq(D_4j_KQxvht8km@a?J>oxePE{uZ6TMCX8q&OhYv$HN1n^RI`_L2~&0;$P9h zhl|vd#_#Vb{tW=fgw?e=B+9;S)T;pAKl`w$<)qigrAl8?_V$>Glv4i}vx$e9Lj`#|UAJwfNF@ZYP} ziX8s_G$13wH&pT{-)`|H|~vlSe-bF}ER_s}^;blQtf200ddQi(sK(dCp! zTg&Y_mK^-ak!IlFEkksUBL`oR-U>dRNPl#T>~tWfJKXEt>-%w{(~+E0p@U!Ne?IN4 z$7H7yIrx(`?s__kPG@o&-_KphsepOh{oGk}GCg#~YIo$lm>V9l@_ zeK~EOTO{|-3FPc#+tpokP9*0Ha9}t33MXE*QFeNW^_(a=S>%jz=Gs+w{@t>35;^>Q z&JvxI$rO8q!r-{y~qSIIFbp3J7aM|fcPN}nI_{Y~*bWYbgq5EICPImf}!>@y< zi%z!Id4A1fx5~}{(a9E_Gqg^V`05K~XCOHxY`e}7oinx0wVhvXAv=Ra=SLt zp6HyfbsApJ1!6tti%zcAX?Q&siuL4*&P7_M;q_ch4m_Z!?Yc;GhH9OL*K-Lun^H{uGTsYuV)N7{QS9EbjE6(hSzhA zSkG9|8K-p`Ue9>3o^hgct=4IHJ=c-L&!1~WXM)ygcs&!z;pfi;(YaphG`yZ0#Con5 zoqVm+@OmbR_2i4rjasMS_1r|xQuerUqv%Z5It{O93OW2doGdywYn_JIGgYkTX3;6o zIt{O9npjVP=-i@p8h)NIogDr=>K4(Np>-a-eL1{@(0rbGtLV%So!hid!|S=79R7Uc zHqj~c(7A)0Qg+-6MdwZrota`icZ$v|51rZM@cleXbnf!dxtkpR{O~T(De};{N35qv zbnewU4L`4&BRcnr&Rnh2@Nu6<4nOX5Mdv;bo%!VO<9?s$6np3_AcybgV$r$ZL+1f< z_;J5qbQXH(EE1iCqVu4K&SG--etuANN<4HP5}gv!dDuf|iRe5mI!irtmXX8v^HR}y z#6#y%(RoC4NMbX*hp|hDB zzJE4}&dVM;uZYgeqEqRi^C~&~@vu^KUh~k|BG&Vo=xp`Sd0nh$tLVJpq4Op={QP-C zbgHyY!;fQciB6U1ysdQ_ejIy;9Ddy27M*P#I`5LhkJmQQc~9%~D&Ek**V*1Dhwq>F zM5kKoG`w9Oh)%WWeCVO`k?4HLI#-W}C+NP9$>|R7HG%)GTHobu`T4~s|{tk8??R+MD+sXVqWji?)jsySrJ{6rB59`?>)>9)opLyu~j~x8E1g;rx zpNY;+51r4+;ok?@DLP+x=zJ+UUx-f9L+2}Us$qNF?MjNy*B(0Gh|brdv&%zgH#vO& z>=K=CJ#@Ykoo_{_)!EW{bp90`d`L-6Y5aJl zkWIJak%+!}m{`=p62$ z(^7N}7o8(KbXt+a_sU47MJ>v|yo@2?upA&aG8KQF>IbETHTl2tgxBn?S9mv7+5Z(9unbakAf$W?_4&OgnqI0sgo@2U48~D7Sr|6t4I=!^@G`xR$lf(B< zFVQ(gTTjFLr;q5IB07F;Jq_=l06BdB_(dnEt*7Dr6B3=E=!D5}&*$KQWA2p4D?-li za9+SWbPJ14lpObb?sHK9{@Aj4KF35SDmoT91uzRfwmfm?kOt1fI63@yS)y~Qht6r_ z@cn$M==3FLinE^1EiP){^U8kY)Pn=>(5n$IBU_bEb#RAkjHfbk6e7Ih&kxwtvnNog5FHb3`Xcbk6nA8B7k} zKj(_hc^*1LMCUxwIp0I)0&@8NIbU>gJ#;Pv7o3>BSA z$#L(eZx(GBCGTIu$l?3vQqjrN*0cAsYu}Td%gDhOI>I}2%M+d93zFz_(FEBAP3Jw_xace(HZHXb0s-Pv(8A-Nsu!W)&ifb`A!|a z68=Pk`8Yp{9DE#e*OL&P(d69abQb;8@e|p(N~~wJ=v=L>XVcgLm9jI29KL_97M-!= zbcgl8vVAvv)3Slrd0s=#R`@u)L$|S_Gme~X@X%VZv{v2(Ja&X%Z`*?T*=lsp&B;n)m4&A1(x$MqV51j&X__;Av zbf$Ue+(J$bThBDnneL%8gPb2(XS(Rz>Y;O+=-etgw|nRmlEa^i-Yzr~=-eebvqk4_51k@%`2M+DbnfxcxmR@V5uG_6I&;b4`)7{m%=6H> zPju#q&U_D@Vsdt|^Lf7LEb!2|Uvw6T&I2Ag3(4X8=K;}Kcdcu8*8I?;Jvbk=+5yg*JV%meSxZN2Dh@X)CcoeiS1 zksNw{4KwZda%4nq*Nfy-!pGqqx@{Dlm&n1NE5-Z8ft)*s$j&BmO5o#e=OxkEOwLg> zi_iDYC%1hiJ1>*NkNalPc}45A%pKOiYdk7N=M~X;l^l5RqFeT~Gl$9byhct181N3= zUKO1!9y(je;rELzqVu|k&KuA+Mb1{XU2lrcTOK-Zi_Tl3^NxqkHgfpm z{5zuau7}QhqVulkyzik?P0m)ff8H0J4?J`}6rB%5=OYiDkICWt=OfYi#6#y((fLGl zwtMK*kh7KTpY5Wv!$apY(b*w7|MSqd7f~_96V^`$Kg8Bqswu2)7f)eD%3p z&tK$}!pGqqy8S6Sf0N@rZ~6YTTMA_706F|T{9APXA*b>C#lNESkLVoK*5fN}GES}s zzdfCr!tbXCMF;=DU24h#*lPS<_V2|Pca$Cc#X_kmE8*iXDcw?%1$$K_;xiFo#y1+26N%( zDX%p-4t^budA~>_r;e?sx#+YY2kxA7`=rP87iH&ga`^sfAv!I|anGMlZQru7TLa`^t~DmvZBY5ePB-O1s{y_@KqK#u#`vA~Xt;8JGp=MzQe z1kve1&iQc8#l)`$Z4YIfIAaB04b-9ZPg#q7(PfIh7o~f8wHZnuktb(K$_Y`g!P_P7dEc{Y0m~hfcQW z^cS509y(``!|xXZL}#Ff&Y7Y!P;>@)=$u6kzrPO>owLd5?A%YE9oF+_d4JC#hd&;k zEjs6r<32twp8mr%vU4su@QH_do^X!n3?`@X$FcLs;oCJ>bcT?V4s+qH;pb!Llf$or zLqz8S51m}Go(n|hLJysb#Ck3ior^tmhLXeg^TncbiHFXmTpGT5Y z3Uj&p=L*reQtNa&VfW**lOTsbKfF?OMv>z_&z!JkL<9G)(d6*^=_t{;iX7M}bQ_-E zbcI~c)#Oxy;cnMeqBDjZd<_h|;~P5qf`IIdC5P|lF`{z~IgQ`n$C1O2`!%97-b3eF za`=88FFMzG=u9Aoe|~YD=u9NXJ%2Wo_v<6~&-LW+{WDQ?Zt$?4d~*2x>ju%8q^;+- zX$1r1dTu1AlpXg;qH_~D9l^oR-*-d~UMoA3$>HbWO`Hw~$lJI@3gFy4IEM@!W z4$+yZb((y9=4#oQCDt=jbY^Rv-KQP7N_OrNo!O#ux7JDY9$6|oMWS=J=-i`qhMm21 zx$N9a4nGg?5uG_&r}dwoHSqX6S9Iox&OEL2*suHG;mSN-_mNY=j@LZVnXh%Oxc0EM zvQtbBKY!+n&H}9ydigGR*fZC2KRJB=ED)Us$Z@ZO`!nxo;CJCJB!@rFKOj1b$ms^# zj*nxXA9c}Jay<`{!ylg)iOyneJx}(TGGBH|$l;IAi$&)lat6U-;2YGwb1oYCx9mJj z&X3@@k9QA=&JquurR3~loh72PjGSC>=xgg8hdn3P^9VV$tg}pX9wi5V77w>;%d!U> zc)dg^IrXgbsOT&wXPC2|FFyMjzGPyaKaY`9$2!YJX9YR#b}euB;#}ETNzU)AvqE%M zkuwyw39l(l-aDtQ>^x2me=Yhd(RqR#_v_(;eZT)nxGS3bc^x_Y{{Fn^toP7)fgHYn z){D*t51k5f`1!LzbT*O`g{^|q-go?!JMNR)^`cnMM$vhR9QXY3J$vHCva?BaUJ{+n z_r>}z{?@eh!>{*yjhs4g@G${zuZqqV zat6c4a6f+@`nG|ui*6O`*&;fxlXC=40=GMt|2sqOpEt=S_0b?9Bi3w}-08 z;pfkrqVpCx@PmbPdvx(J`{jDxCZ`$<_xkmg=)6NtS8(uN)$q?aZzE?NeB9m7?}*O3 z-+1Wk5}j{EXE!;m;9P}weEohLahJS*eM?RiIPT-iZqfP9 zL#I}Bz7w7AJ#>B`r<$$jd(qkBq4T5Y>=B)xJam30hwq=CMCTU|ojTF^MRfLh==@3! z-#>dr=Qj_X-$my)(b?yrv!5KkfA)#aA09gOqVtF7{OO_d7de&e{P|OK{`Sy0AUc1G z&OhY*1NSYwR(`TAa*Div{Y%c1upak%`j6-wB&RzxK>L4w1`~&;rfh|eyVsqAqLV^S z2XOFq(*3hCwDKlmu!)pCgkwvy{V$plpOe^i?-|2&ToA! zJI%=9&l8%8PIGd4I-UJnM}8zbY2@(z(_C~~kaIh9a34P1dS9vR98S*fjsySrT8K_d zaxR5~g`NIwrZ(_($Ro(XzwzXL-q=!fT6yTS7M)h2)5b$5ot$25J#9qiNDrO1qI0C^ z9Oa>NG&y|#93?vKJapQNPCL;##zQBA9KL^!5uIZ_bdD39V@0QfhfYUw`2Oi2I-NXp zI*U#x(dpu$lSxiTcK&n`ovt1_$BRx^(dkCc(Rk;B+jmfy6$KA*IlSHQ{IVU)sdB64@6BnJ+$ms}k;qBSG7sHn*%=Ppohwq=$M5mvJ z>av6o)Tn}Mn=J_xW`Yn#~d29O>^+0wSxR9 zNau-mXHAMbyWb%vHFL>X4%yelC8Hd2X!fZNNp0#n4?E;pHR=N~s@e7S?1wsWL52xZ zD9BntHVg8;AiD(l+aW)wW6>ENE%A2SVeWQC9Wo=uCFeOLshrUcc~k9;+Z^(`BFh}| zc&gi}a0u>=@hSJ7Y@eAly?XNzbMW2_N7*55Rp)QvbjWuD?p%EZ86wCiK_&}QB*-#B zHVE>OAbSL90$(SzZ|<7$D~EV%q4r_eAsZCAz#;3^SvSsg6uHA8e=4%fA%3+FUvS7D z%K6wK$17*ALoyXfztLV{XGH=IIdq1fFUSNz<_PknAX^35B}mFm_IeJ@-a!yckYR#M z7i5JXn*`bJkQ3G3_*FPL!MC1^TOHgkk!ihTR2_d*{pqY?*(T%WUU%CUO0CNg3pTePL&|v3(|6`IjWB! zg9W+PAuZGuWR62Db(B{+WUYFn-Qtkdiqtyf2}Rn#H5zYEDdKm?aP`=GfkT$5{WHZO zz0~ZFIV7T1_<=(PDDtmEu2-F|)9l$#P|iSyB$PARAsy9Rk2vHwMc#JE>uS{R4!Kx8 zjvaT4J=c0gvK_KakyS^?u~;KIZwXQ>$jLLz*+&TSutS=wJ^zt#Cf{myUKFG`{&*SOs@0ma9P*(eLmbjh zZTn3Q*`}Pi4tYy@g)QiH|A;@2Xbh^`?{ZRRIhn%3cYqD_W3$jL#-Ga29 zY0i}+$lZdxD9AoR0<+BWNQd-PTk(K!o)Y9`hfGqVzI4b8i-L4!Kg1j(6E}{no_2>Rs%RLq~A2ATJB@tsqCy6V^z7QLyl480*7o? z=g*A}c}0=M4r!@IRXXGv(O6})!9I`@@>m8DJn7g0nIV4LRWqh?8 z-U6!py+isb(te)pOi*i%J7lCH*E%Fal@~f>i*jCe$OUSLeU8RgpywnXI<-DTjQm_QpF7c}9(@cgUgpa_9N>?1xA|kc$MlUXTX_*&@j9|6}Vu zz-+qKH-PVRMhSw^JEMy_IuVQ>eS#1XO!RKlh&n>_GDNSVcSf{?V2JKwqIaWrMi+$p z?z7kVyq`1sd2a6g|Nig#R@rNpb7q8(BoF_z_gY4#%4+m6QbS~sk+(9hBSwCd%qt@w z=apj8r+pRok@`mB%Nn+Wgrea`83u`Pu`GfFxmiw_ItyfkpHH@|NR=rCv5!TP=?L-n z<2o}S!SPsjL4xA5Sif0TNCK9ewj^Y!3bB9r^y4ggLLxqA*#e15$`T6+OU9CGie<$p zsRId2&Y7Pe_VInbSrN%5HgKq>wxr)6D&}_kuoB#YbBE z$Y38?;v+|VE>3i(J~AL;KS+l<_jtKf{0{&H15Gg4Wu%GC2e?Y*Lpbn=nu zKC;(G-uOtth)??p@sa61@~4sfGUk^?ipx1Ixxn%q6-9a)`9^j^gprSTi6cJp+DGy( zbSi&r)x<|e_{atyIcX%L>|WO*r}AMLVNoL=`|9K)OO2fL3$Xk=1qVT;xIV~4WaGXjnpPv@d;8jAq^rm|BLhU@FLy?mRHUqtoC&;j zx)}MmqXzlN1Rq)CBRh@EkLRs?+b5G}g)<(PjHjNFkG*&Bk%2xk&qx09k$biT@ERsq ziQST&B^M-INd<_14$gFf1Sy$n*HN+u5~1WNBw9(fRTxiBu3Q@usAMoCGB;-yK%(=q zoQ6c?XL$#)k6@Q~J0Y=39za5aIP>{h z)KO9f;$N9FZS0Ja5s+vlt07huZgs`ZDDhi|I#oGS5E4?I`C!+n%MuWY9i^lmBu2?_NN7E7wFP1Yv)q7$D@nT^E3f1$NUV|{Aps4!&NxW4 zl5G&H8E0NYBAT=0`W@F`dzRXeP$hjKRtL^3goG%vwmOlJgL&D`(JNQ9Cr5UUs0Nw^u~>CF-diR#1B5E3|ur7t9S2+IOUz%Z8GcB|nm z_aPCZSW<1lcyKgwdGAPtjr<{Zq?$$s`*|}Rji5(7g+mI)b@v;MOz0TyYdIuZ$q7i% zIL=sqU^SE!v1L4GT0o-1S*Al`CbOJ}SW{UN{fToroux7)9G?Mr1&n&bNnWiyBqn~YKo66 z_mOQra?wX#8>!{;jydgar}y9Gyaz%;)ETS`iC)j=vH zTQGu;@(>cUk0tBhxIZ6d35JB8V)+FUbB1LHB;YK|8%T(fQhO{bLP`_Vg&5SMo+R5P;2?UiOm#`t-quMvA^c69BI^A0cxnec?% z*D6S?lB2eK#+hdj|3oYq4q$YNS;|8~Q?PV{MB@*2ew^0~NI+VaD2RQua(QQQ(a1iT z>1!k7WTqJodK%#$_Dqk(InytZiOtA;eGduF#L@>6m6>HSBu2>zh<$X7?;YVANNiSa zmEjO(nw=#O;-8D9zMaX<(%H`BW0?pE%+IpT&J<*cf!N3QF7J4r8`&W%_Ss=i#2PLu zmfeW6Vli>A7%~B>uLh=7O{}xqQ*tELab%f~?DvsO|8vG;_wKa%3KFLJYHv#+p6L`w zgpys5s6fuVg;+&dN*=-WuA~zrOv!vml#-*6@Df}n1@3e9(Z=N+b4eoyWSyHDnIP-@ zixFp?pBIelJ#!T@(WSVr6Oh2tEYIu={xZIg*G`&a*mX*(KtjrMrUxXdD$6WLa5a{{ zAu&oGKtii?CQURZUW27JB=T#Pu@L`SEPp|QlsvKP;ArRau3_STJYXG?HOy%wT-LCX z5oZk(6pCvNTOt!un|mKUqExH@>^k+iPNw77ElO%ZqQ2qGFi2=a zmVJ=GZ&{w(8T@N)KAypXCorR?EZrb6-?J=$gf?fnY`1F3lKUiDg|K`F32n_X6%x>v zWg{e{JpYhwmn}GcaCxuOaz>(Lw(GK--`oLN+ha~8Ah%t9XXMo5T~vyiYw zoOut4P?GHeGD|p90TRBJrKK&AEFn6aOO@T8ZmV6w`BHT{K|=rH%99}BN;cS; z-JCfEv5#&pZ|~2H9G97lac?pS9^3^reInH(3K%$gPh6JDF%pO}#vpj`_D=By#b=*-gj;mXZ+r2z7bS#WzMy zy1mlX$SgmvOfcf?%wOZ&nd|IKEcbN`5@N*9?$pYB3;%e+nbMHhr!0*i{?Ax?LqcA# z%!35JV)+Xa{V&T6NW@!~w70PuA6QC5B3y1OzU7^1D@d3h%P>f6e3rG4sDvyhA^wS2 zUP0`mc6@K;oOc}QE>hRX-FV(ieY4bbc95y-iJd%lJa;KK*EGw&m%sEJ4 z3YNDJ`{?cRjyd@Q4_Ieq#c~>%CoA@q5og7=#JOVMA`_5``|1h_Q!?7lq~XjmNRX0) zkdSnou^!?|$;46w5|o9d9mGEy%X~;k4wj>k0Q}1zK3;M09^qb>o24McKQBuyh<)^R zc~`8Zk@K=*y^Tc3icK)$tk{7x`fN7htk2mv z*XJ}c0Y$m@7p7I9%rwUnPv=%rk$Of_iu5;fcF z%n1EF|Ia><@M6xG9T{g@{UlfSCL=$KTre_3#Qn_a>sOKdMn;Q#Yvkj3A8BN~WHuU^ zD00!KRqE$XU)3a2&&bEhZG2>%kL)!vN$NcE$z*-u^!2gwcRn)M$O@@5+b0uiB$Z?e zy!^EC4@MTq`5IxQvFyx9BSjK;ckc@$Qdh=v)5uJbl&_sS zWu(;)M*4^>gG8JC<6O&oAmJtXD!2}bRO0>@cdb&K$pwk3%TgT@-HD|;BxoedWJus> zmcJn3V_2?1Lc>|&y@CA3QUnq`jpbWAGn=IsBqD-kvMtM5wm_^kEEgcLn^~;4SdBkf zvO(;lzsq}0OB%T>=e@R(#d1e#XT;u_9bFj;$92aZicHL3+}CVK&~}!ykPs#5-g$;& zJ(GP{$w-)tr?Zi)vJ(~<`C3L8ZNwR2jyOm7!DP;(m2>|o@SoHB8<8ePew6X7Hu9}V zqW4a#CL-;Ov=rH9q_s%e4^FGLBHfLABeKs(Q;`go%hRm6NEaiWMgBI@P$ad>Y1L7r zqmlL^2aIeJd2XbGtbAtttjRt$$jaA*gqbtwtYHWwN}bdGb|#9?`!q;|lFg97U7R^* z*HQ8g61I&T z3kkW;@&hFBDa&t=C?$I!!OuAJ0%9M-UEZtmv-pnOkaLmA$a2~9#f><7zD}HbzBV#3 zs;^em%I)cOkdMspk%K;BC2&S)_wLjwWhB1b)q5JrBr*>Y^qlAZH^l!X%RNY-lI#gx zR_q(jw1fn`W0?lATz>QhyB!kaW_b$Be&!X1{ql?XRx&q=M1)u^9+ta#y=bPwHy+vWIrTI$sI_HlEjIf5svye zLL+x%gkKt2EhB7V#2I0?I7iqUnV{S}!f;50lC`$v;mlD;l#)k~kbIm;l>~Dy$Wjp! zT!y8KovF++4q_i;WTuOZ+>@DZHL_M_dfJFH(*beL^r@Zsn)^!nImWD{B*d!8nU;_M zCBq^9wK=l^5?P1k0VJdzOR1#JyvE7AnizQ?^Xg|LQsyK|an5TGGEohC(j8(S6XY74VC0cpgKLcZF4y2cMx1N#&p5Ba z7s!OJ;=a;nz>ZQ<9}=r%6vRGuS>ET)Tq8^5nS0yFG7-OwPODBb=JZCkOP!KNzLZ(i zH{#4P3A0G`FU2Msgc(r zLyX*!@ys@|OIG8Sk)AT1)R~>$2g;ZOjZ~Epeq+QL;fXj$*wbVNyS!&-x{+&Ad99JZ zWjsfWbdd4V@%4{GM-dH?%rqmlWHt5}aaQB=B5{2y^3-JBNS*w7 zo!*B_U#*NBkrA#o5+HMrHBv`awhSEhFzmhC{;DxrjEIk}{sGfli&& zvOZrMnJnwm(@09mEH?7Jti}l=&T5p6b2Z#WoxajaopMH|NMF5;oRSglG?Gu|9n+3vV_wr9x{HG zee{7us&lc+WJ=3;l9zN^Ws>zNYh;?NPirF?Bs0}W3t5dlMx52CALnX3Gnp(>r(h|k z_ZiYxTO(&=gpo$_%iLcW36b85mv-tny*G}t_Z}wmSStT+B#oSl$435-02$|8w1Y&ebFtNAipqHWzcj66eJUFnBkR-KNIc1`HPS#< zea$oSkBsn%kz6wO3ST*O8cFZNjX1q`iL>`ZCUaLR zrz-E%NiOH&J0pjsuLvWRMf@u`t%isUH4-D|Vvi9E8RuLitccG$>RdE~*hg_0&w3+C zWqqz2nIP+vCdg@(L^Ab_G?vvEVZ>RD0dcOzc9Tgib>15Zm%b`ia{4+UBb;m`kIemw zk?*AU+?AbHPVc|O*?VV`xi6JB8c8AN;$I_2q_0X+;rLP%AF3AY58wr%T=d0n==_0+iGvf3aaLnloU4(pw$oQxsS|8ux%4&K$Q2pkNh8H%?iuPhbwZ{0CPtjz*T&iVRFion zl}{VVBIhDiU8nLn>8qiUV37?*#*3t`=d^kx=c2xmWXL$@Vhki$or}vRQ&z@PqP|ln zhpbOKBeP_EW*7;O%yA>FWHk~6J9V7Z*dFI<1er`OsWa5b9O-Mfk#jP_G~YOt3(DME z8EGTEFE`@!z9-J!ADPS(sa(2&Q#qZSi-AT?NMC!5)DkJt&}lVNWQmbGaxU%|`3xE7 zToh=8&ll=k41w53IT_C>BVWk+B>dLN%#-ygYb38^dKqaatFh9Evl=JjT#c(HQ$XtE zXzcVAA$_$razRG8*+^lTyI&KhRR`(4rV*$2^Kte*(PW-U<+DaI$hpY!om2Ug^wri# z9g$N;MvGK$>a@Bi=VG*x#K<`3;(w5EbuR8f0@v{qB=PsSc9ax`gskUGFeG3jOMghj zpDaruLAzK^K!OjkyoAIaW69kNeH~}12Z=t((i>tQ;c^DY8i|!NxX{Q(IXgRyIA`Zc zoM-1eGC}9LuNTzH?di4b516USG=unG=GtA;dnW$Zqj(urD>H(DjzcPxm3Pq#$jWm%~SBxwWNz}oq{GGI_YhO`&kQ3orPU=P_U?7o zxj<*9avj+l6^(q%d}k!DjPNH&a0%Yw5s;uVEQcW>N@DF+l{k~73$Ds)ER7)HU$cya zSoK-{vRi$_@(dEun5A%6v}(rE84?x3G8+=ziRG-_ss~HjZn*#SW2p(T2D9{o#0+8C z0tp?)^4!jhU@6fZ;~B>?7!n=MvIY`4ndJe*n#xil6my@(G6E7jgJm@&b{5M`h&7ib zR}YsJwwa|B#6G6U-WYD=rR=e#Mz+eee8h-zhwK#R9r8IcLI2~vGWK-(>ScL%W>+IU zM79~JDDoZ>q3UGo1v$c%D?_4`bcTc-=ge$K_(_(1kf8G{&mqDwewXE16BY(+?%`xJv*nl`!Y#TD6cet-}rqyU!jWm6*e^jOvB=|13Y6=NeG6WLw zfHUhL_AyIlanQ&cnZK6okaOSeirZW1h^SuaWmMi-|^d%PiIy zab__;&RHBrCftwvx&w(-68~qs!6xKPE=aJFE)e_ZEYFm=kg!DD>W0a*mrUyZIA2LP zQxX!AlqJN@Bxjil2~5ed3lflqywLLos}Sf)Y30$BDzqO!9* zg#_nhNjngGJ{L;`Tk^23x&`s?!IE<@GCf(ELZX$-g4oA8 z^zQcjucJnOkR9$G;$#ZTn422OE8jwd8@VFyNXLwPE4wanm{VuINF^hUrPX{Rzlpqu z1oYxr6dj7QqogCmKDx^+68wUhnv65A0+7%?P8~-YK*E*ug#`EG%tA;=f0n(F=m9LT zkcgoy`G=veVJvkZF-itNB8GEj6~zBnmVY4jvC!o`JCBXH)G^}B;$@t(=!8u8eC}%`Bv#1+NKgc4wnIXcoQBxPav9-WBcI85-WmCy zNa|5eD`$is;v8W?WFnSv?^R5zaLc<_|2C3DE50 z5a*5^VkEW5Y$F*&P8-Q8l488eQzxfL9V7WfMj9z3vfD^8k@rR>inN~K@>HH8GRw$x zk+Vi-i6ooow3;hY#YlumZzDTJB8>bkvd73lkyl2}isT7*c}AE(qz)t)e8 zlG$kFg~&rA??g&Wa(OEIxx8m*fRV%^hmE8V$u!w%m0qN^k!&KXjN}n{Yow4!?cba_ zB}9HRQcmQ8k;)<&r#P)@iZnISKxB}SFp>2}){5LPvOy%xRHyPEB0)yBi~MBdp~z|@ zS@7oI?C=Xl>WcVJb1DamR58*}Boy*{2|lOuAOY$u--A?5&zXYLUDnqbSo%UvrDORM zvO%?qH^XI38N!+35Wg^%_K>gRv&@7fAJ1|fvO-CTnJz2uIL-`#^ii@OQeYxyQp|E$ zdzG|;RGGk;HIPY4UO<9+awg|&m-Q$MOD{;~(kzQ1nQF2ehWs~@|*vb6HtyajOB4EUNdj z5Wh^ENjl$U?as_n9Wvq!%K%6wHPfw-@9S~q17y7Fy;_9JN?eyS;~@>|upEZWR+3@? z&bw+=A97IjH3G6mov$sB>Z7>w3&>kF=DZ7C);V={nnNb4R!bl`)T-Wu3|Y#RGcIyj zrL(ep4XIR|Wdx*RB+GtC%5^LW7rU$!3t6f_>fdA;47vD#WhrE7eU?*@Q^72LOI%i` z{47Nw=ah7WOj6hK8c3at+{(4oWo=ZMQjjrf<=aBesx{mJ*|UV}yo4lDQev6Q`e6`f zLLoWSOjknc$8hEG)l2R*NR>$R>=>z#)$$H3J zB@ZF9Q*x_;;nXBMf$Lmfecb>H~{i%56&!y%u}sSLl%c}CdnF?bwb_QszZM3#hLz) zFO#xth6Jo(Epi6w9??%3H`x>g9k7>BoUy$ahS^k5hSD6Z1T-L%=oEZ$6tnReYkap?~Ey*8vb}Oj^xvExU zAY}JduDk`}KZ4~J1lxzMixPk0j-+-%XGr^boS6r){}mG-ucm0ohu>IoZgp8Rr?3oy zq*IxHAZ?Z8*oJqBY22z0WRsF(km6H0lk+c^W&h2NAIH-hQezU!PDsmemL%Kpo}i== zWYJ{K%!h1Kb?!qFsZ5C-F6;3uZZ#4zdN#{dNJ5nqHXRM?AMr=B}~AvIKH8|1ypJcnd+ah+WI@Q$S9M@T~TM!f*iS-sobgy3bF z&U=den3w9k4Wwoc&TNLfQ`hNj$e^{HDSg0Y? zEhL*dUo#-_%X6J`kZE7Bq&M@v+ygjMMW0dJX7OY2l-Q-og0vsHMvfb3wYa6BP<7rRniC2OnoYu3t1D)bq+&r{mGK>qRV=? zm8CA^mu)N)AwfU0oP^}Q$P(`oev+kTQ5DinWd=hoDcKEa+L!Cxhn%U*lJ_#^t}=BY zhX-+HFyx`Smj8nM@HuDVT|p})jUiozaAqjv7j_4^udd;9N>Ogr6LM0??~p92)jLSZ zP;OP~IzC&?W9a~St#;H*NaEF;*>7i-u{?ncP?;}o;O%rHXSzb>Z(`XB3ERw);wGNL zD$^2T|6Q0LpTZH466*cuK4j|#&Xl}`PZO%uP{>eq58ntmq~tN=W+JXr=r%qDs1de? z6jN_0GaxO_aI3wLHfLF$LV9&$$$JNHXzEVV7*f9~XC^`-)xB;vh&CGFOfH0;H_Ea{V9R?czSSss!2ifTb(s`a_oAAoDA-9EBtcVoClG z=RGS+dB|yesQ$RB?IGK*uq=es$;5IBl0>~HeEtaErWWT+9mwYGER!MYcCq{eDY%m* z%VU={{2)t9$ZqxR${I+I9h`Xu$-AGWYAmjTJuKmn?MGNHLkb>cDftB7Ybluq*;Rov z*B}j)6ncvHRdv4FKsxzztMQNS(X`)Y&Tf; zLBjuFc?~lIaz`g@|J50EsxjvKdlkAB*2> zXXOvbI+ru@TBL`Od~WZ`FMt$O>wE{YVG&o({V$$qYMpyQj;U2$2U(|9^$H}}ms}^s z8<$m0t!hQcX0@ulA%)edE`*U9D;(NWH(g)fC8KwW?Pk`_!ry zd551ns8t;Zv5$i;?}}|S@~_A}BVWj>X8aF#Keb{VATQL4Eryg(>vI~?S*=gJ_xO2} zTA#9zMrwUJLl&v^Sq!P6*5@{4yIP+NAMn{+txr?PlPF%XrH}_|eV#)ms`aU4emn1w ztj5nq-iWL-l3&*63MA7o?kkhaZGEp+qb_8UT8#k^_n+J<64GC-#x=-6wHle+ZtJ?5 zdtJyZHTUt5bn32t9+Fbs!}HzK9;><6ha_sqD>f7orRKg1QeMqHSpv6JM$P>P$OSd` z6%hOQpUm{Ck#{2L5;~cJGWQyg_G<3KAWPNUH$g6`xnGCWQ*%%MncLc@=3WburVY<@ z7^H}r`zFXOHTM^g)@tsR@ppQ(P;(E5xYTp`62v}^$V}5DcH}>i+C~b=+y_9$tGRE1 z{7=pOF2wI9&mvtCx7AzCy$>@d(o;b6Y>Ec?ClTsCoSY zDW>MN8)6^FWXx}jd=Lpt?qrI{EJ7gX)GX#e(wF1$9EC)wG5h)BN>O7j2^p!z+zqlo zjX47Hof>lt#6F^Bgz543hj%>8mcea(IiBS^$VnwDAPvTG<~bxn$yXWOR^y4B83Bn^at6|E z0%vmI@2L4pNhe6_o}8HhNu8DD4y1BvmJFG3?bKumf&`3W=?(dAHOmRey0a{AAsf!K z&~)&5t|t5hQUgmI7Jb)}I$xet}e~#j*iXMfGL>{XBUxai$vN zW@eVbkQHZGHb5$hkq0d6Z&^*#tvs4i#rLi*HU@yqVEb||R{xuaT*ggj7v z?SNEG&UIcvqSg7zm&0v!ROhQ5BkQ_NsD^Q?bX&`N zu+)O=Qmy(y_J?w24dkV|3hqEQ_u@?ELT;;hQkL%^Ro1gihg8naatbn0J+0ywc3a^V zXR1IFRA3nonO>gdG~}han&KD1wWIcYF-R44PTN8%tLuF-#8P{HFJ#9m?khnc?g?rq z1VbX#on(-mQL+vau0D4^gG}$umGczE-bla_0traSG7mEIGnRXhDXLYDVvymSX#tsY zjAbcgy80IMCgclseWfps_rm~g)d$cEophCt4$d-xVeCzW{z$*!bi6+8!BT&FGMypn~G{OUdP z8f1cc2gp+uvrv_PhIGlnl@CL*&1ZQB`8owlu4?$^Bqhr?koM|)O@TxzIRp7uNyh59 za@DzL4#}sUMDrkje#L!VfIK_SlBNdk>L*zmKwM(*H|Alp4yP< z>aAcHq@I$kkW+ZfeY^uCu8EnJWC?;atjIDD(!DFoT8OI~%WX)|VV2^x@SgBLmT{1` zDsusn_7G<>)OK4vj<7U={HiibAP28;<~ZcTB9?e{@SZt_r8;ERXqM5CJ4y~hE|29* zlDc@iP<84-s;bNc$d_BV)o#c)iCL_Axcenx`5Ll7$t*~|0i1aVsnMUMM16b)`IeR8NRGqz$EMIV) z6b;-~J0y)g3RL#$=UO@I$XUW|VZ+bOYT0-u2X9; zmFjD5)d7-dE6ZYt|2CF;kR?hAev6s@%$XjL-&F7GA(Jk0=03!_#8RZO+j>)(r8C6; zbCwm5AxfS=9x18P1S3@Q>I3;-U#_zaGV~`F*LPTrek`RSzbY9H8C{GsJ0ZuF#A}M5 z@2Ge6hLE*Ixz%XMDBewo3ma-i@ztrXi$zCYBM9!dY0hL)NH$ z_#folcFt63h26W0r>z!{DLXi`98zmP%L~ZEJuH>lU|#Au zI1y6+C}(a#b||UQ)@|LWz?t75eUw~=yt~htH0|(>kUvWWNJ2G>UXZ_|ITHo3kAGa= zZ@*s~ak;%xxV@7pD$)XydpOsb1!+@`<$sXj-?6-hoM^*Rx&v0MAxlTdVzq`VAog)w zMtIeTTgH>FqmwBnQUg+3jc^!bj2hu4$VxTB+mL_N2m?Cdp4o^;*Z^W5CuBThjQC0K zQAUc3#6luf?>Rc-Gm+}O5hPn%?rS`xtm=Iy#6C_+@2`x+lfDXdaWW-DnnO;j-X}s{ zs@``&s;S=JKw7Ha3wCuXpOW6&7>O@^%`#F_ir!gb5ova;coc3gX+CK6bz+5bsA?<6iBp-}VC8}~4$gwk=SpZp7kL4ERw5n5Z2;PnB za;7)rMje)|kYuXPYsi76oGBEBZvwNjG=q$aWSIkLwvOc@r1?UY^h4d&?3*kNA-Ny2 z41heY&$1G-AwSDm$ba!!lKz5kx<<3KgCte6m-t>Ni3=iPUT*%KuSq8kXUZrKwr=Lhh$x zi9ZJ4fuv&zg5Yh0l@?j^n!y#vsT!t)~#hD@# zu^Mw(et|qwauKp~4rdC5 zHp_pI!9`e#Pj*{B6=vxG$+41U5yVej1=k?UOK~RCZ+K&LvDAj(*NN#H;8Bp^%q#~W zIX19lnu7aJ4wkNvYV%p5X->C~U) zHstKLEa~UsT(n{dhFreHG6YiVGRtnrn!GF@AglARl%I!hCiAfjft>n+Wf$Z@b(SZP zW;Iv}%*Su4sORYqkb(m_GZo^#!g2_5NY!}*DYBI_r6Tat(rqjQA%~UhfQ=W}KuBw;m{V~{yY60E?TMD6(+kUJH*)i01H>do_SNJ=%+Cy=R`xK*x| zZmUrimS&I-+gTPsUhiUwf%MtQQg9W%wLZww4f0tu%U_VKJ6IB|c3YwQSsFr8?PZw{ zNpO_qA*7Fzs%!B3nQAq{A+sxRs|%0~{w(p<;x{CZv6O|}Qez$gv5yNb@3#;;j3kjy z+0TuX6Uns>t-j~VjUn~hu}p+?RQJ_gb}JkgrLTXDd@hxXL^_$TL|Q`fsJ@Rg18VM2^2HBx1Z-OLk z&*$`!X?0a9=h@^`P9b%EFj7fm9OQnSVJpZM^^MtT zNDd{hAU(slRe^2zv_77t4isa;U--G_M9zE%DI3nR0uo-6RoVfw{qaaJV-T02^G)pbW^ByceL6WMsxFwKODsvv)R%T@rGuUm!8VS$05H2C%$=%pb~9axZ>M za4<_dNOT&OC6NE6X1N7Pp(M{fe1g8pnZ}T!*H|V&hKyi|f}Ebm@(j{z3QP6L$XieOqqlDtuyrv%uq<{*_=5E`A=J4a`1BSM}{m^8exObOE=j1-Y!gT^RuR@-}C- zKo%Eac?g+Xm?g&%d?s4S(j4-|Dwa8rqor6*K_<9Z;vdDejGOky-><6->6@A57s%J@ z+m%z0KXPy;?=gG>JfGz|$iB5K%OO)zutY=Zt8Z8SgH%-CoRp5nwR4hN^@hwm#j+7{ z@E?{3kUzSzWc>%;*%uUR457J`*OZk)deeC`$!y)g#W!VI&+lu8OaIvXUfsv-pna8*bGBvSTC5YDlupEdN5Tsohfe96lkcPprKlW$tmS z&5#WDSzbY`2$sO}_}o;Dr8%T%b(VRMZJAj9f%I3OG(Wq5@6dK|rXD2gZkF+o**jUz zK?+B+ zj5#kmnQ9^pAp@In?5#s=VK{x-FGI z7)d8}O2#;qYlyUk98i^~L(+BNSwx#wcck)XSDiZPrA`GSUyF2uJW`b-Amvr%bEef@ zshr}PQ#pgwsbQq1$WM^eKkx`wKw7KH*G#K>QaR&wr*cNA6KtfGNEqZxRrz zBt&Jh+`wHzWg0_9sLUwHDV5m;S*5vUlXUPlwpYMa#Qa_ZitG zbN|oC6Om%KoL1Ro?mt4RmgLG4Av3~Q{)U`VpE#aC)|ci?j@!7FbF#FA3{;;l=0I}P z=FBz7{JJdp@8B#iV`&LFsXjYIK$fY`4l$4w3%OP1yZC0{CQB1YkB2NnAm!9&hqaKV z>XX7P$h;-oD%U-Ha;e190}`k{DMUgptIq}3ArDS-tF-s=XMEJER)G}n%b6aKvOQRq zLh|)yIS)COlqKy0d=q$y?{)@iTbeYhD=V#Vm-vSbHB2D1&Iz|=?B?6lw|{C z)L@nyke6v#az4VJ8%@p922xtdVo2YsoH+?;a*ZYVV|;s{J_S^O6k5QUpCR$evaE!J z+-A82Ii%icGsoh0SJXT0H;`^Cxz#vG{Z%Z7A#*abBzb~X>g}`zWNQk}jDYk}@0lAQ zZPk0`L&)Tl+$#T5{M7XnODD+Ve^?emem~6e2$D~|L)LhP&jsooavJ2-A#Qa8QtmoS zis!hR)H`Gq$ZGX|_!H#s#N28VWcmP>&tBk9wW#;Q`jA|WITHry)r#d0NWXzBPayTw zyI-D{_;;A7cfVGUbboQH#gJ3#P46zGj=FXVy}}!Q6>ilL@?5>eZHA2hhBLPz_g1rH zdyQ`bH?p*blu++=k&ydqIP(PZg=$sqUwrr8oim}3LF(Oa6QspG&b)?HRPTOe-{8CD z2+s6`G*NGQ8z5IRapo2zLcQr_d5cdI>J6wCpHwD^%g?H(GUXw2Ri-b*J}z0_H;^SpcDuZC(a2Mg6mF+g4!1WG1nH#SK)OM8 zgz+pELf)!pWi;fBdINF!`B_<$bE}e&T{&5LLKdobl68=;)jP=x$d~F3B!4_V>y3H? z=?pod-auACjxFTAZbOpZV#yKT&zi2@Kx#pLRBs?ZLzY$IR{J20)!Rk<1b$Y$LYyfB zNqL5)Go)n?mIz3_-Yh2}XR@=TOz3C*q22+4AqCYtz&Oau0B&^va(^g`-)DZ-+QBR( zAaky=w1Et|#xfsrSUt^;LK-gMOrk`7R(|y~{}M7nJ_% zC-$@YsHb@|$Q1SLUJH4!mRr4tj8#wSGD)z8>Pg)Pa#KC2H$l=I=QXOYQphruxdIuiGAYvHzN#{{Ad6LIAmo|KY=Rt8 zncI*IU3eA&>HIAFxNLc!JN1qH?efY9BhN&(8OiDPX6`|5gmLAp>HVziy;$l%{!`DU zL68ILxw940tv0u^GWb~`%UDW6o~fryAIJ;!lvxicuAVXvAlubbCUr(XYovP0RDm3+ z#C?TB2Bu;;2Fb6UGVV-%RvGn_DGB*6JGbfuxtx$?J*0tpf;@zz_=Pk1GW%I42eUMV zJWdUX#!1-Y#5;VJR2q)2j_ z>okCrQ}^&OkdfCpvjwtU-NWxfUaNa}mTZ1jsJa{1gX~my;}MWO>TbLfvR~be-$Gie zyK(sdKkJ;juXckpQuoY7kg@l9%r_uCBUrLz_p`#)-R}oT(kRX>hkSN`g%E7KyDZy`<9T`L^&tP*D~L4wu&$NvkAFb8MqLE?SJG7i#D-E9s)N~^n#Uw)hm zb+;)2NjH+~w1&)Iz%mcgSKV!nL5{1tO_BoGnd)v+335i=TLwVlskgyDAVt-^MEEI$*=bOCCEIr=Q9-dvzn_t-v|;gge(6FDX;eY zM#$a;oOukHqxO723EWrJ4sQ%Oah+QYgH%=fa0}!wwGZz=lCX+P{I%epE%s*#a>F7Iv` zYviR!l#x7cZ{{)NOSM}HmhrP<)NbhsIj(lgGD!PHJm%|=C2F^1EsLk8+AWuBn|c z7*ez~x7rLTrFOz&$SJiGihkv1)mA&96~z5F*I57=62%<_@tKC;B`sUF?$Ehc@6Nif$10PnPQBu0FxE>8%zkfQGUmNrDdFvXDHd0^O6ensysb)l zTW#@*RNmw>(=9&ouUBO5_U&Hx{>qz?%6Q%UB!d^?PXsUVj>rh{^7M(|hVn_PB|d#E z^N|%kvdSw(aohDAiM;1sRu#8~Px@Mgq@L*tugI8j!}(;)YkcH;?^%{sbG_q{RmCgC zCw-Olj!;&$l2@coH*X!u0Lt^^J&aPpPhu>xl5~PpA23IKWUZ5r?3C{ zWRCjCAs^}H9igmGGjC;)o8Fm<{OQfeis41X8KDRs^PfmUZ!4)&+B`bc`O$e2@mMJnUt+$Xbm?32mu?Oj?G@R1%q(#uEs_(*dfN#P?o zeI&b&wDgf@-dU8!mhc=8@K4XqlohM)GxuuVjLh_jkGOo!PEDWQGx$h4AKC65q4c%W zN0RuoYV9NOea5rIXUvPdBKz<=?{y)210PDA^^x%u^{HImD^jP6cg(V?ExaOkw)8%g z|Mq62ulnA3N#zY*k^9(eugI9c^wyEyfAWe{?(LnaWUhHvL+Y&aX*Jc`m(;1@BX50L z{piieEC9}`dhU>u@viYv>`*BFBZ7BM%ZlKPXM{nRfn;Pn3%tF1GLdm+7I`y8i=lE5 z{x7Lh)Gp+YGxJgGCs7&vpC83;6|7p>i}($8!N-g}iy+m?&NN8)X(mLqvZwkl-sh!G zG5Z}Lgj*H!oUlZc@r*g*^G_uzDT!x&!jgR|F)W^Q5t#hb3|kreeBEb{iIhsJHbB%_3jb--(oE*=D9ASQ<8Y@b8Wy!P zF{+haXJY10GeN4AeZ5c2!WsMi6QNppGFg2xs+A{`%_qaHWaaDmeAe6OT|>$A_pX>d zEh~a6%Q}zro@J4)J~F^ZlKP1Ar@8nO>wNEylBb?aw+f({dU_AdPU4wGIEyFapMx?U z3G$IpABpghXdkh1QsrXyjbm|clH&H+3FM4tj~&cQ8PEQSRIThGPRUQ1;`YR%xm9sH zGZo(-VtLH9Y-b|4mFElstze(?zS(VB$$1~B&pTI^%ogd(u4*4`ymdUQY8B@3cmlY; z$q4bkEh~V9_AY>Hs<^zL7oGM-iuJ`&C1dFEPG zsFg>eS&G@EBdT)7mXPWso=jK`63=)-YmyYR?;8*B{h_Q_G0!Qi#o`%s9TE?O)%{c= zSv-Bkuz1!dv>vtcWUOEkPvvkQiBe*RVp;4-SOE>GPHDUJv+r5#Dkp_K7d_V4kj1o=oPi|5Yv zxF@yp+}R?0GQqt+%~-uy>=iqP*Q$?mw_lE}eITfW&il$}DEF&rTF)q_5lF%Ejz8Y_N=5*?Toe)d__BZ#ic? zyG6CKGvBQIG!x^~D%EPv*k?I}TX{Y+K3GE;&-EU_t>jAi&S&NS@>%&`)XH0Hsj}1= ziBaftip$v%S+UvP6%!fj-3cWT_8hBxWSx&}_lne6<0I>RWT%h#`Fw6F;3L2Jw5sK^ zV!wM=%rl-4ULTJ{ucsCB+{YemX0e~Ju|Amx+dj=$TUhKp_B@Kku4DbhVrP!*WAS8m zk(9AB0Y_QvOboa3WPUvLX(o_cd9IzKXDH)&BMRkKp4Euqj7I{_QY&W-Sv){4HC~RPTpa$D~GC9_H|m|K4psAbz-Hl~LG=^_9=4ch4QNc48KL<^2<}1Y)kuQYb7LIWZapW zGM*=6sA^@O9nW{~?{iYd^9^t`x3W)#<)4c) z_Vx8XFN=MBMdv1QTIKsx0`rr2#_V5^L{{~CpHIjS zy@wT|-aS{rAAuw#?7K~b%GhP2IOD8*QEKIpARh^1@!X$>m!MXjH<>8Tcy6NMoRRfu zkPHdm!gSvwQLtvu%|GKVqt#z;UoS)g-ReoSp`S+vBXQ}d;uw}P0 zWsl2uq({0f7yl_$7P_?0ne&1k>J`WQZXuCvusGCcHTriCG+K?a(50HgE~u(03SF8t z;G#gIS2qRZl8WtjaiGzxqR^#TA7Rtx*~t%G+H$|%fky8YRFF%v`UaXLD|97UICxdb z3SCLoWq~HiB9~+xba|jjvdAUtu*VgFMqAD&m$b9nm4Qav$waQXR|gvHPae6Xozd3> z8f~X6bm`bH8B$e~4PC8dK`*&6(6o}4GvpGo`>;TxS>>Tiv-%FNs>usoC(2%Q#mGQ& zqExIXbhVb9-4%BR8oe7<5xU$qI4If%mq**+WZDLc!j^SwDvwCz*dSf^HmBVk66tf^ zxPa)`_nQzBIqVgNF4cTDv8pCNbZOQ}*+8RPTqU_=E)KwhVP4Pa(4|?sO|7cQ4_%sd z-vfb0v+~F#S(i);G&=7kp-Z!lo?cax30=uihBVUhgc(&`=}5C>W>rl|=+ZOT*(HHS zkKL(<1L9UouV}TDMXTj5S}mp2GP<0~do*ZS$5k4U@`zMKq%tI~hkc?Rz8Up!9Q82u zSk?YyB2xT#sFBKF{X|G)t(JtE<^?z?jekDSG?yD~#pIH9MwJE{y-Lc4F5S9^z7%Lq zlw(sibm{%Ws}=+reJWH;F6rU;MS(`IiHbs(tKs0NhUHNWAD|j$!vSNaRRV7HV`4+3E8@qsRBc(53w;j5G^Zg)R|gNh8OU(lwz*wv;k* z3E8|h)JQvJW!VaP_BCpK2q6p%~O z)0?VlN<&v`(RA1nX!NY0Fm&nDu5DWbjb7CjhAy|Bi=u63Wwf5B)3%eU3|mHb@VZM? zQu#|jbZvC|EhJL;yk7&NH9Y2zkVxe-k?Zw81C5^h6_88n^U&XcM(3hDbZLFg{5Q~O zeM&-?_IXk@e7^?T(`%4qJ%=vs^L06aMzc!DC97_2^+2QT6q8Gi7tiexXtd?Z(52@R zHELGXl#xr?`MFkAOCmNH_XS4;8okSx7rE*;3N-ozJQKP!Yvhrk zMzZqBCEHGoqeG3%MK*NlIm!HE1C5@OmSfaCB8oHDeHVxX*ecj&8Ln0$B3Nly51IAld6dr>*} zlaD8|d5dN*{9o31A{x1ev|pt0#A@nA8c#$s1?}L~xGGtmSk2*)#uL$efOd|GG@e+^ zagoLmU75GF2NfCKEBKt?=QS@8qR5maD zT!^YE(UmFa_}#0LlPz)Uu$MjVIM*8yImf2E_x&ESM5aPNZLG_+e4{1Wa(Oft*%O1= zQCDe5PAI@ao_|iDIiUcD{Jd5HY0(B7*n3wI!H%_zTykR9Y;2%8K~C(-$t8QOW5x#> z-9x5Bmp+@g_3l8U`@@3JrT4Y2$Oam{uaym5`h2htHWidGnJ;}lSQNVSs&>%>fkvO= z6@@On&h0xR(CBq;dFWEjl#;5NqR`b`o^4H^6KI+jU^YrZmtI%?H8;@cb!AED(r2|B z7X%u8R-1Y*AUduYuZ9Fuk*W+`dMsFtR}ox&^jJ_Dy4)JOCEAOAZMOoanN54q3Tj!F zZhA?WB_tCP*Uqo_FT8yf+bp;39BS9Rb9J%lPSY=H=Yoh_6p_9Wxhx`=N2FUsE{+IZ z%&Ri4t0U4qB6!8FitCDq;5CLSnn6*;Ua}R_`jmy0S5jOS%#QADN+MG5MpaE&L<(_J z%dIGFry?RnZ&lT#@VJ>>*^o4s$J_aD2bvbLHRXL6kmj;sl;RY{wbM!lmi{gvx(#N2 z42bry;FqeTEFy)!1{$~O#zfo2Mq7EOnPXR`oTQ{G!gjRsLaU?Gyyg!klHMMqt1IvK zfao6StG_}bqJmJ66EKwL^^@Vl(8DBJP7fopy7o?{AcmOEg33Rn_E&E`8Q7-;w0n(P#a| zp-ZoccH&Q-okp*7)1k}F>0Qyh=yXD;<89k#6fDTxXG+}ovVea^=p26`f^r7YCwaqwS!tBcd<%FKkW z=5hhn0e{NvG|lA#EU!sOanpF_G^vv`A%R`NnEEBpkYums;qwiUjM6N5Z z3^aO&A$3JSbQC?V4v6-rD0Hdjwri?tGNDWR^VhY3M*EW`myF`d>jF*EpCJL!tZQxv zNYbCsrJ7kcR@D@TF5UAlx+&1;xq4~n(suT}InZcUdE}ahKelpPoZfxP4_(Q;;}3JZ zM(3Sevc|5%Yv)dr%zNazwm8se4~s*Wj_aOLp+m@bMf(joig33|-ov z+wgLWOV|EnLYLNO#-u=_^PUY|+RkrN0*$s)8oBPfKhWsBr*ON~rR&&hJs1)hdkMK@ zTo*hPXf!J?a-BOP&}a`cp-abgHEy^1beZ=|=+b#V?2$mDbuK2C%*Be5K%?_s5V|z| zpT`1?_PK&wa@BSVF06ciWbAp52PEmwQz4OcSWGUdPt&JEjm&#N=+gf5dA6!19lB&# z`nfkcrr<{llB0~%yhz8CqOfH>o|Z?XG9)r|{k(yH>$$NET|ZBt9DKI?R77ywA<^hr zx7e28A9cANo@9II$h%gfJrvhq+m0g*?3I*rwTs%pi^^4c_=fGF>(BMJ9oO;*o8?;W zXj^u7gl@ByclG(kx?DSN*(^Oe$S4NcEJv=gL|u7vgL!xDw72cJ{+w#lo#q*v?lkSJ z#*ypoC>%*!jcW%tJ*re}y-jzTmbTBXouRg6SJgkQ%T??!doRwl^QCRat*G-YaV_6x ziMk5@vpyYcmUDHA#`U}%m(z5x8b@#_tg0ya$d6FfXUihE&X=<~aYe%$QjJD%)n%}L)k+bY59KlQVRq8Xw zw&SYtp-p!r-}c$H)77@@>hrC2x%%uG)#qK?j;qh9mbjL0wZx_0Lwzp#&x##y(_K6N zk_-Qz)LnL@PP5Bu9KnS^l@(PjT2bR{JFbTx*>p#au%mD-7uuFx#lEvHSFydKioIvs zaTQBj;#wYQiKb`&vpy%8|64oft>^grv<+GaV|!O^%Lu;X%?`c~seA3F+1 z4zU_H_Gz{qN4~J>jvQQO0a1NEw(Yq3bg{&>e5WOvUjCo;X>PNe zYxihec!*GCZRA>wbKwHHil&CuxUo;L?KtwWO?RY`?XzpShi%!_=X>jN_1Pz?&j+?0 zSD(`@aV_6&iKbWnXMIkvSvHuuII7PVwjEcWb1iW#-(!iU7yM^^ zPP19g)jAs2lXhHAbE4Haf?FI_wy(BUFFneDS{d5~?{)#q31a`mYn)#o$Yj;l{sOI*w2Ez$IB zRG)%*!5&Hfj_m9gLLyBUMw-3nSJh-fS1Y*>F!9wu(@O7JhpskqpTF4~fu@agu_Sc0 zk$VoOzaMI(KPX{)C6lnD6Kw;?8_iZlvB+%#^y^d{l1H~YXZ%Q zau1|Dbh*lBqqR|K*M`%eAjyi-c~6He>wJ|&ByVkyu20)vT^ADRPeG{Bdqlri1R8xh zP!_uM9?=8e1R8xVlUg4VnbTGqLn3op61wzn9-Dp_X!L!Fbm-EjcpHBVH2M@TO)eS5 z)XJ)w%Fw0n*ZuZupwZP^61rTSCr5LEN3m75ZTEv_HrE6%x6b zQW&k{{x z=xQaCQ4J@VPSZ*>6`@P_*mj^Iyb z5|@-vKMO2z?bNdEIL!-I~pbhoI7dq%9?OFU|cJ9Fr1U9N}sTjECX z5?bdTW^50gYe3}M5Rr>*y6ey5mbm_0VqLD~2Q6{^c{%D&|ENC?MXs)PCAfBuw&_X= zaYGm&n4L6Uk90(j`04fm(X*XQsA(xvSlTJjw3I0<3|)G(8jl+eu4O#~DGOanvY|$g zu=BbFE3BA(0}Mg)V&$C%jULg; zLYMaWk;#EZ`&=HnRP*E1s+v-A$r`)wfvTDmHe@$aJ<~gRT1fOPE_CV2{BnAzkx>+e zECx56dD_u{B87GqmS^35kp%RT&WN^ZVOFA_XcV zmz)U?{5{ZU%jKab;RMixbOWJwz&#Icd(53sj9=iff^3HbXay6V8)ex_7R;ghb z*N|F9cT!~$$@~+f>$tjM?Yh;gV=oRhTEp+E1sa|AOz2Y0?KxF7m7&XxYgRO_f1+`{ zN#n|fE$iB-j7Y&ALAt(Eaed8@$T6-k)Mz_WqpLA^-BQ%`*q&8g#bG`Uiv>8AU4a*OS^e{~>30rV@ph@<$p-Z#6G_0yABbT(Zs!>%a@$TxwC$wqmf|!^Ezx-|4_kJtcOH7d*B8C)Ht3Ga z&1^f)b#!!J6o+_U|;7fgjmdkGl>Z4gjHwHw%Q!X14 zJ@YIa9%%G^(abFY(Z9qgzcnCw6;LuNAlm1mI|8D2jVeN-nvyYrMoHeC0dXV67k~ug z!XQ(-**ZJTJC^7S7KiQV*efDZdRLGon`x^5*nsF)F&Hg z^g1OIx|+)&Wb{LUrnwwKipVAXk3T%#JToM6 z)?FHDHa%QbQ%Ek^PVd6fb^Aj-GFFBz*XJjraqVtL;WW5Op3HkPuCV1~ToFl^1nK&X zZTCMF5@{$CYFyTws6RDqy3;JTMEg?`wygId(ReD0fBZS5kUcHYtw6{`OI*u0TH?0% zlOxh0B4T-(royEenb0-0$*x} z{b5eE8p;1B_4l8LUHuHsc#_W^r=r04s3z5{$h*3zt2X|dD8@FN@q~9BZd|HKg;{4t zS;y>dvraND)lk-1l=WAm*T3_wp2npb$~v2}P9A#a!N29&tQ(C>HI&trvX1<^_d@TQ zW?ZVFtaB*q(|KK9%c*X&UNbJ$P}aGW_3>%jc6isf#-$p{I*+nW_`dq+7u>k^IW3tl z)lgO;WxccdlUCk!u5qb`vgCJ_{Mu;P^P)vR=)OeO#029~4P~8ASsS`vai({@VqB`B ztnTW{L(4dbr#{PnVa;v3*&41lF4a(056Y_Fu=$nVRV|%VLp79jfx4u1>||2s3~TX_ zcjX(GYRGk=x{@|iscTn`$?>k!jY~D;x=3Ai`sfNbYlg2gj=C+%> zYqW8xhO#b>vMO8+A2BY~P}U`s_42{re(GIIj7v3?b!n9Ku3PhKj7v3?)ho*Ku78Y6 zHI&txvfAgqKkaRob$I(^MpQ#teJJa^1Lj@iU0sYzHI&ttvR-P}U&Il5u(0{l=vl%DOVjdc|eEY+S0LtgE6d@A}-hR6|);M_GTm ztly1GHIy}&vSuCIXP$Q*)FGKK)lk+olqLK50=EuNF)r0m*0q#XZ&Sk+-qpvrR6|)q zDC@ocldk>4jq5h!QVnGlQPzbGuIlYwvyDqNlyx0t$+(Vi<9g4yR6|+UQslyy^-ewkQhC)- z)~%Fv_B&S{`h;7DCmEM&C~E{|E$hA^RC@HCpA%)0*zu;cnwn4P}j?taf)4-RNB}8JB7(>rTp&bAUE(9e!zCs-djA zC`--(ylba%sfMz~QkERQo^|Kg4bDg^uNul4M_F=??Oh#=OEr`=p0Z?r__WKq(zsMZ zS$9)byDz4_=v@yOmue{Mo+#@nm-VJ`sfMy9P*(3%i}8ay_Uvn$ajAx~CPrD$IoDxb zk{YUpvhJm<@zd^q(!07Cmue{MK6Tmal%Ez{e)lEr9J|Q4R6|)=%G!IG- z=WFjj>s_Z9mue{Me#(+{*u-sLR~eUTDC+^r+P;2n+PfYxF4a)hgOnxP*NJYw@u_jC zhO(wnmYg4YSFN*>YN&>?9*VNsyR5WvsfMzqQ`Ue+b&m6{8;wgflr@90WWKI-^YyfG zsfMyc2YytP}U=qCFdI% zH?D5Rr5egAp{%x}=YQp0cN>>#DC<$m>QMdo{tetZeABp8Ls^eeR<(||yy#uqjY~C@ z^>~zZkjpxP}Xe9I=;`g>fUv+ajAx~o}esQ^KZKvPB1RjP}Y-_HMh;m*Sza> z<5CS}Jr!kjch`DbjY~C@HHWff`|_?s&Pl4F8p?W_vgEq*WtVl9ajAx~o}nzcuJo?q z#-$p{dX}6UmR#$3SKafHajAx~UZN~n^AEcD>R?=|p{)6owX9XkU%YFiajAx~UZyO$XMVjq z2UuWSs-dg}l=bg*lgD}2D&tZOWxb*EIntc-QsDr5bWAQrE$7$YTxMl97LWb^X)4E2mph4b@QA>&_KtwKXo) zP}cvPOWK`v+k!zptH`)iLs@0c6=ywZT&kh0H=HZZ`pUReLs@S+SDdxK{81@Bs-dj6 zoGZ>c%eYiSSxcQO&MG!8)lk;k&J}09VqB`BtaqF%&icu?R6|+sI+tXP9d<*1KVOZy zC)H35Wi4~AIO`JQQVnIj=Uj2tB;!&IWxelQan`%Wr5eimz`5eAe~e2tl(pQs;;fcE zl4_`ivOaXKIO}TTQVnITaIQG(QR7k#Wqssaan>s1QcbGaO+y9`?LSx=PnD}n_Gl9_p$zF4a)hSCm!1che8OYoT$e zhO)ks!i_ z`SPwE#-$p{+7xBI@3IcMD5<<^DC@f@%e&eemue_$v%2hAr95rF)}6s#VO*-AtnWDs zPusm~oN=j!vVKrkco)mN<`|c1$n~STl2IpjvApX8<5CT|wy4W)%W{QysauCX8<%Qm zTtCsc#`Ydt&AaN#mp$O48p`@P%KFw_-JD`vs-di{lqGA!yRI}Y)lgO?Wy#&1SKYX# z8kcG)Ya3<#b;)BLylb&>sfMzCp)6VRE!>@o4aTJ!%KDYE_FlAij&~h!aZ*FoP}cS+ ztEJ28Y+S0Ltly$6?;37gs-djkDNC-h_i#0Q+PG9hS$|O0V|T21)w?ztmue_$2W828 zwRXppLoZ3Hp&H8ild|Lqf_I&7T&kh0zoM+RF6&O?QVnJO9c6jftHz}o%Gw!ab#Pf* zj7v3?wTrUWmDg+MU5DbkSLLG`%K9hD>g2M{GcMIo*1u7fca1VG)lgOnFUIkkPXD^K z@uG35hO(-u%bo+w9o%lDcdaun)lk-MlqJW?BDdezQ@*+hAJtG+4rMLtzxG7$Dljh9 zP*yHw$sNwRE~|%esfMzuQ`XaG)L!6Sql`;6l(jo$9XDgYcMo(~&l#6$C~FVOnsNK} zW#09*ajAx~YB-lvYtx6njIZOe_Qp5$%11SnRnxgd(Wvy(E4{0=ajAx~YSFl4f7rV^gX@n9^+CCW!0uExwEv@jcb8%sfM!lqAYnr=UsdBNvfe5%G#T< zWE~#q*8D}rr5eiGhq9J6D4XP6_ZydLC@U|@I?iQ%VqB`BtbL;_@7kwtGA`9n)_zge zbuQ~{<5CS}?N3=RH{Z9^yM`Kb-Q-<$GD!_pLsmaL85E-P(Zs-djnL@F zcfY;sN8?frxsG-{l#xI9>DWo$Rr|7}8mb{zW7k7DQvLbaSry)Of^n&aT*r`0?tXh$ z593k|RpVIalHQzm^-I5c*J$HX4P_nYT+&0%nrB?9p{yp(6=!{AT&kh0rp^^-)f$l0 zP&Jg*%(>#MHpZnIx<_=py23kwKkJGL?*yubTm_M9YvfW5xtd3=%E+Y}szwWS*=tyN z`>3-!-?-e2OEpxD6V&C-MRD!mU3VClYACBEWy$`qliR*Zj7v3?bs}Zmu=%aKylb^_ zsfMyzQI_lvx4Eo+FHdG!HI&twverGn=QQs+(YRDYS#2mw?z2^M*A6|6OEr|$ma;}R z{rGV2y2rRwLs=(LmW=C6x9?qIT&kh0lPOE~^WOElajAx~+C^FQT~@P!Nexv)S*K8z zJmvJRKE|aQ$~ra5s^_vM8kcG)>$E7#yIwUe)lgPC%39)%DI1MTHI&sp%JQy#u1Kn( z8p=97%KD$nYHM7op{x#3mUj&_F4a(0$0%!0SHsE1r5eiW6lHnWTgIgt%IZv6@>dd< zxvXD}OEr{r24%@#a(UN@gOVDmhO)X)mYj=LciY!Z#-$p{I+L>GFBiP)dE-(IWt~M? zvVA?__8Z?Cmue{MY|4^1B)zNlmC3kNLs?xZORn`EcUdirOEr{r4rR&pmv{9wF4a)h zx$3g#Hu5H6Pj?UY9^+CCWt~S^vOn~$7mZ6blvSv%@E)vptuijvkgJ=zl3FGAV7;r_ zRS8!Oxz1OYJ$^|I8@uDzvBsqu8drB3*O4Xd8+cb=<5CS}^`IPq8M4P{+S zS?&KibBcFOH7?aq)+LlB_eJ}-tk;c8HI#KJW#!*<-51{VhjFQfvU*XL+>biYt&QUc zCpAvNBQDd2ZkP zn{lazvieb$tV8c=cui6b)lgP{%96G5lFKSIF4a)hWt1gv!FiYb;*)$-LsS5xCs4P{;9T+(j$%Ub*@XQ zp&H5>7G-%?XX8>0WerzXuVuI+f=kkdW8ac5R4|`m#-$p{x>;Qf4#77L;D61^eeYxI zNu?ezF4ZWW4q)Lf&1?d8Z@8NYI(nDXv`$00KSqyR|k?%p|n{0eLc|&tj z!-;eRIS?cs`v@I-WgYxSd>+iM-je(;JJJgL7uC427t+{=q2)2K`1D6imLJ7CAQMRQ z4~Wcrq6xo)Vx;ybJUe~VZE+{wm{eCav@&l~m()%a^@pu~-n(uxF4f?_;@hM^rsDtl zK5qso0Ezo^d(@xDU4QD|l=Me6)Su$0Ki<{DxKtzEN!<%F6aUxu=WP&Ky}pKjgUlyQ z%V9bCdu85r0|++hpq;rOD~bF9B9-@BO6%b{sS47J1d(ba8oLt8qm__$E5T|C9DzOc zCfbqZ`9%>=?gQDM$eSPyi0pPtQZepxS=8rz>S24h z2GLe|H%J+gl_06N!>v32)|^yhO1}c62aW4FkT#_G9i%;xP9t(s@+)3`r1ydp64?kM z72|Q`(J0!E#GaOVI1nVC$W)LFk)he}U)w?C`}=*J4=KjDXr!G$`O$w(2AIvHdOX>I_Kbgy{^WCv-!0g-*U*VG@4Ed@8m zeSf-v$iA1WS{kkS$^li6UlkE4k4Tv#h)9h?Rv}fj45TNK9U#4k6x@Mo5V;a0|J`tQ zW`pF>xYmHk*!^B&pE20tNOJ~Ab0T+w$hUX<`YZvF@3i)0w>z;CT*YKuok8UH8@=Wp z5ZN31%6|khoHTXr!fGK+PZ0TiIp5C1AoHEZ?jcK}*(r5(cAAJ}BU0=LB2w#-CBK#A z)0>S=)`qX)EgO-Id>A~(xn#qOYr$<@|P z?uvB@+L4cHsA6NNV)90`cda!p)zFo~os=a{yV5y!cK$Lh)lk-5lqGMA9XB_oM-VpGv7mZ6bH1_e7C3hh2$H20UI9`< z2Z6}18v3!9f|QWvZ;+>nbbSEF0wNEBlo8nqBCEw8 zg*!ipStiXa5UD(mJrj*R?Z$3bLV+XUesv;kgK{L==C1XALzaA0L)+lJv<=FWV()4* zEpe$v{kMRuqES2xBEL`?@9XZP{>Z+r$nERCG%c%!`je&p$iB|Ib{UsyX#X>bvSdGh zs@u;W`cN_o)lk-C%98!OcXcu@)lk+H%93$i?vD6Z8<%P*Ybs^Q(-ZG{#JE&LS@%Kx4vOX~`)lk+0l$GkS`Cae&%eYiSSr1Z{94`jC^>g_2q=u@YtZ9@bZz+4%8OEg= z%6f>hp{$3atfsr$^NmxDOEr`=E6Vb&fySj8%6f#dWF2mD$J~j=r5egAp{(OB@4m>p zUNA1zP}ZYS)-7&ZUTIvap{&O!OTJsxyLK3tYAEY*%93sQbhj-ZJTs}hYA9nX~T{qqTKZTx0js-dholqFB` zysO>ANi|eMSx-kPHI(&Il(pSuye@Ldh4}JT~c-V$luzRXyxs>u~5}e58qIiov#;m4gceQH(w)6mTIVn zZ&DBCO;qoC$+%QQS#PN;oG%d_5S=mujeo@2M-CugPw{j+~uT zLp9XH_o;_6U*2_rajAx~K2TRUU*0vwxKu-~!&}C{b-t-uRWeh zs-YU{;Tr1UJ5Q~c>|L#lOEr}BrMklT@~#2Kr5bXrRaZD)-gUomsfJu%sSE$oPeHv$ z`g+%L<5CT|zE)Q_U*5ILxKu;cSm#`JzP#(?IY|vwL-SRku5iAlNAsl`>fw5I)iC3d z^WGlr`fIq!QVsR+8|tClOYyG9j7v3?wSltany9WjkNUv4R6|)CDND}Tyz3w1QVnH& z8)emTSp{<24IkA|)}|=Sy9OJVYSjM}NClc2E-od@LGs-_D?ybYn@LmenVeJ;(wqWP zN_P~m1zAYs5s)$>AA_Xd3-8KSe-`)sD61uiJZ-<(^{_ukjoh%$Q$Y45@;*odB7cK4 zCen6pa_9CI*YZsuElD#UMDBtn8haj97~O}=A5ir(7+MBWEUy&tZYnlI#}iiw;Bf}djz*7IE;^4^7C zKW~7{A>WFScX2VupY0;!-LZUB*YMSOh@eHoP}O)n7n=@h@0C;_QW zZHSBkNfUVs zq>M=JLbO9YOoI%f`rHcAg*2~%bR)70q!*EsUqw4a?gEj@`}&lF3?}dU!PteOGz^oq}bKajy)CCr`T1)YKj~QpA&tjN6+wF;#~KeK_4JX zKB}Q(?q+q#Ig%&_6z$yNUEdp*YUud>eUvrdoww}1B*{_@W&J=|a^B)yjf_h*l=UNJ z$$jHn+_^-1<5CS}ZJ{i=Z|q$Ij7v3AmegG!ZlvPPt@|@ATj{INJL)B2$Hp-H5-RiQ=H7?aq)-RML3}O`7E((@3)oq=@(YZ(>^}auP@xk$xa`sO39Aeye<{w{NYe$R z3u#7zRFLKckV4XY3$leY2fmH{A=T#skX1w`gH#at2&9t8Uhkk9ABIQab3l4h`Zy4I z%hb>En;=D``4{A7BB#HblNwLtc95Aw-UE@f;g7=mEyMXUX?lUkEPKrpAoEGH6-4GV z(b$zx;Z}krg_qgu0!z{nDR4yAMw|Dr4yitugDfL58Kj&@8A#!Zu&R~P4)yuq_mlT> zeN{VwtRl_zAo5L_ev5koM84_JlTSe8JLf#9{Xw#NeZ@L~W3^ql)Fb zYS@b9IU@C0h;}kG(mz0oh_qgg*&%W>NFMcPfe_rB@zwYRq%mole~5Q?iQELzn#ju_ zX(H8DB-j=7KCEvKFMANbQfX4H7v;$VcJYC<5t9eVzr< zfU5BU$N|b@OtcXZ?M9Lfq&$0hT4|1y6`>T%~bJL%N?i{-l`Xe9J z&~b35y6ic2y%#^I>0SMeOEq-N-KDPZ{;GG~W?ZTv*FWl#Un|}X|Md7|eYtl%Y+R}# z*S{`HMv&<;agleuX}sTv9K2KImP0eVWuzHI%iR zb4d^7S)_NhF)r0mR*rK?yYekY-Zj9uR73Z2bJZpFlaD-$JYQ$o?xTdNhFsMnSNF)J z8XDK`>cYSDBX@uoxGSmqOb=B<vd@>ddm-`f>r6={Zm$bD|_ng$|QtDY0Ila}m zR6|*NQI=dCd)Hj!QjOMmEr{$hd^L9e9Cv`J&c}nuI`x`MLDHnT3q<I=mPMU^aCwm*-CzSNJ`vkiW z&pINTQ-8QtQ9F}B4x~~152S)Ln?V|pCT|_;Oq$Lh1*EwZq>?nRg0v&e?;tx!)4T## zcBHu)B!$xhzvY&I^dQYjkQ$`fdp*vmNpl8B9%*g_8BChTKn^6$MKkyqL*xt)+1L4%Fb<>_Y2F8seI2iaVmI%0B@{U#D`C%# zm>oKQ?g+Aj$Vd>mU5J0{Kls~ikQzj`g5(it{4MsPMEZd=ATk}K5s^eI3F9Kt z8Kn4=us@?g22nL$1`IbL{8`|nXP(u@U>c%;SANK0s32mOk332Ns|konZZVIVt6^9)ECY1V?|;$qhCCGxf>e{bc_Kstcb zCe1JqoZ1CvDbIk^ApD?2$-w9!L|?i~{LOnnfUONV6TJ2Wbx9ftE>g z4oEN3j07np&0LTHr1?r*q}l6FoL!KnJ;)%^3wZ5Lp0{Ch|Q<8IeQ(#r;F-;kh94#*5#+MuOzx_K4qa%mdkr$af$I z5~-icO*JCY6GTq&e0?4ODIm>fAhn&wjy)6Ar^r>qX&jOI>|ZT6RZb)A46=&I%^(#- zo&(8z7OtNSAnmA!b#}|u_kVnSI)d~d%`lMOL`p#h5!nDTlt{yzT)j&2_2~~%Oq$0) z8b$RfjOtV9s$uJsc0}s49_>t^mTTwc>N`t*T&+N+lBOR>1+9&HKxUHWb&xHjsRWr# zn)=mq_01xm)gELnX|4y6N3dS=7|49md;*f^G~pepy>Y#lBR4+vqwj*cd&|{!NB8hi z4c#@`M_q`}D-n5rqla@fGA`B79jd%2YoWVC)!DdILs|P$mfWH8t}Bg8HI%gP9BZ4Rc(xd$PkcxB6C0*5m^t?gvddAVJ?Vt2T2o|2vYV1 z)fuFaMw;3?H`SHv3^Ir`LqM)0@;peE$af%fi8S2@`*2r7tux3%(kukYkY*RiQqpwD z%hkI*ejh##q?|M>K!#I0`{SkMwWPTiWDIF$fovwtMv$!2*f}kT<~`+R*=i~?(Y0qp z$|F)1ky1xe_#CqzMoKd{7-SWZc_0-;egUZ@(rW+QROza4C5!;sPGesJ(vw!gE|A)L zhg)SDmjDM4!B14BjwG@iqz#b<2jGb2sw(5U8l)p>7JA-AO%-mdmIk(C9A_aH#;yl)r&@PGspm{ z^IIT8Nt0JUH#L&TWgruYl!BBHsdW%W;p!vfx(sA4X_kO2Cr#ahu~m|$2xOJh*gEG$ zbuP+Ot#c+Kg^r**srS%wDb;5;{3*dgA}52C5g7_nPULxz?3%F7KY%QyQ5=26_6~EpFyS(Ir3P{1(9ALr9`HJRDKoC z#R`y}G}4^oFgr9CtwHMG#T$P`e=W!%M4km{P2?Mp9z>49L=?Ij%Gj?2$<bkY1#z z1Q|@4mQ8UMN19os>xy~(&#Ls_HB>`c4Jb>lsf*n;^(n@s8p=9^vgDfDyRJ1Z)#!ZB0?E=^T>(-;Tjeexq&dEM zZt7_wSAfhT@({>EBC9|qL|Z}TKiBG^Gdzmue{M@F>f>rW==Pw9fB?ETwAv1~QlGeAo#%t0YZVkcFff4wCvh+^?2{ z)F9HhC9ao=i~`9LSqrj~vRa&o9#Ym=kS(NH2l5+{Q(EPws_z@_vF`^tkjP&k)7-pE zIr_B5wK-|t0+~&kqj8`tAk74jQm3(d$g*fJTIg1k)uba*;7GWKJR({N-?^1gDF<$R zR6{GFA*}@YK3MNMtZm{_jrz|4DWSO-0Wyckvmm8J)_{}|sdW;LDeJ;>RqRR- zyhIS(tGpki2Wggq^d|Br$S@*}PsaKoG5}<;n+qA(BOo(L^CieK($sB-k&@sF z2r{2E?}1d1CifIngEVQ7Ely+Sy);@I`EK5=CNCnXh*b8m=d0FL;fRd=R-|WXF6M!h z5ZMS)LFC|5vAxqM&H=$`R4`w+fovqriy%J{*#hz_k^Iweu0^CD$d0H!B~g9KX-lStwA0uc$_#Ry zv#zREQyh^ZN5UhKyg}>F)#aSFr#q+3KNI!AM>TXLI+Bh=a!%`A9gRyhbpCTxl(pEM z(_Ud*s-di-DND|2z3U$1QVnG_rYt$99qrC(pEoYmP}VV&CFiu>^@VY%M(gq~$WEHC zqtC)USXxo%3c)=hKc~Y%4j?iIfo2R6|*(QcVAm z3NKxyKFfb$&5Q1Mdc1L|hO*A!Ed08tcXcx^)lgO!b;(hr8vZ$FSc`|e>w4o-4Y|%# zmz=}P=h~HHa=hyS<5G?O_t!w=>8(Fjt^+xcj;A^2O(94=o%;<1X-s4q zNCA<>AZ>_LfTW4!cE|ZVkz+uz8^ZI*m9t0Ucns-6wl4b{pJbm-$qKz-W9U;QUt^B|2X%bmuMhd^H`Bf{5ftB+n5U`wvJjp}E-WLfi)+QUFp)q&vt$A~%7Q5qSut zoX8T8yl=yK-yj;QM(#zpPeSwF2xK#9&H&j)qzGhBd~c863Z{TGBJvi9Jkjv;^$Uo6 zo0fMS(G%4mO*fEsq!|Sw_Xd4Cr665MQvo9P2EAsVi*avhASV);1k#hpDiFE$^z(ISKkWHQb2*3{Prc?@kRsCj z0wTv#uW8dCdw$Z~4kAZ1uX!6}9BFD_mhAn!=6sN;qyRo4Y%F4b@pNCtKGd!$!EmXc;W$cIEuydpRC9g!lC+`8dfT?isA z`x*QfB#$&_48rqn(%cKufHa?j3@6PYS7IMdn#)1PkY+AOOVVry$vTZ)333OR@0eBg zijF%GDUV25L`ofzm2mP^xv65Bi(w$+h`aIRBBM51Ff^2jeTft4TW|EtjC3M%3~~5ZT`S zxQ-r*cb`aeHHd8QUh_OiOVa!TA}iBtPP`G@1yywjNEaecgY+b_0VG4@kee`rL@oj; zA~Ff2n8zEIp`L=XGNL}$Wqcg1(NSHc5PJRsUq*O z3*2h4n*505MI;rG%1iB@Usl~uNYA6WXml%9IFU<1@`+3aDIoGbNF|Zn5jeinNZW$Q zp3TqcH6WWwGaFs%4dMTM)O)s#o1%n|9&b7(nL8Ls(FAT@{_c01;R$i*O8BKL!|p*}AMkz;|c>Tbo@ z_mbvhkc)}j0y2QeOCUu={*-oHHDna0j>4HRX~ux`avE2SsA8qA8dg){h_w73+9{!t z{tGgP$cdw|eG$1Dq>RX8AjR9lI)4c=j{3aM9m$c=SL}3<64DF@d78+4koiP@09i_; z=@{$}T{UD}LqN(&^CHMlr?J(Lw=8&j&${|pO|c`=@^-Y7rI8lgiE|PngF#A&JPnfn zOIV-HAZw|IN8FVh^?ZFU1=&uT2SIibSq)O7ez=_;Fc#Y>kv<^BZtOCOCqeQ_vlV25 z)7Y_>M)fIj)v%h3Bhqr~ao9)ENUs5DL*#LgG?CRHg+%hkW2^i%tk{_#O{mW!LF633 zSF9AIBWX5)oI~V@yK(#?(i5bJ$PAEat{O6~uRunSCjTCsXOgBr$T-qG0V3ZDm>jKa z#Y&>87P=Z*O*$e4j!1vLL(BP8pCc#W%7w^fAO%Do1xXXB0FfIm{;c5ei8yxCD0+j) zvD?>q8pvGId=Bz5k%R8V8w*5wf~+F)ILKmG9~swAAR9^3=02QblV&tXC22kcS?M&k zhVo7qZ{7K>hE|gok(49SpL$uWL#j^?kOPTK0?8+`0;GUQ?Mdh}k***+X%u6HP@UfZ zsdG@cb^il$2$A-ab5l);+y;^+vJ9lc)kns)?-V@QC(WfG@@?Y&%xX4BPtyDhvcqX? z4U3~XS6*VzcC4l%BIS-qe_Bt)I{YnMha*915P1tEk4Vk?Q6D1bf#ef;K*;Z5%j-Y} z(6|~tkepTe6+RGT1Zn1i+(~2y$aEs7K8Wp|$Rvy@zMZBIVa=1K7)0JK;k`{!G&^ONRGppDh?GPm>xj(PC&((Gs@9v1{XCIgAZa3x zf^;OZ38aunlNlg?g!45Nq~f4(D|iJ&&hY(w{Ry&zG^ftQ8SKHKYdlC@A}c_e5XpZS z*Kw|fGScfn+K}cg5P3@M=c3LmjEgh_LAsLW6_7&G)O-Znm($oSB@@k8v8%k*6h$Nx zkwQmg6qg_?LscCIGKk0mkRl=*L5hj&UxKmk2xq4wNG}>!5r|w{`PrEPGL$s$fs7=w z3uGdZ;~vF%6p?`-16;*qTqPiLN%JL05or#13}>*UDFhkeG`6aRQRUOFhE`J$k$gv_ zKX;?$rIht9h+HN4YUDhQGhxyk4yWvruezqEyYAEX*%91zbylbzyNe@*+ zS?5xgyvcKSj?FsOxKu+~=TVltY3W_(7?*0a&qF|l((FtEk?U^1oxTV%hBTjpOeB(e zE;luWNF$J$L@oeXh~DwByWGvGbm&fK=8&rtM6P^&f7XDMl4g(RlPh1ZIRPX~Tft=@ z(}+v~DIu~HB>%5)Yx)CZA!Qvo58D^@=X{WI(u@XKO=Ka+Mk1R*Dv2Cfiv6mqk5r={ z$WGGC0x5SIyY1vh+fJFQs@0S_A}y~$I|Ve-8ZTh~Or#Y^n#cf!UP^Ou56D6y?|>{LQf&dQYKgQ1SxaOXNClCXK=OBm^S%vaCRO9uSCYFB ze%||o%qPvmAWMjR4e|w%`U|l&5$Oja--YYvVh+eo(o}*pB2DX8u`&-0SLW>?@?E&T zop(U$kY?}KaP%b21t9sPc^IUN)7UvpMe|X_u$1i`y$PGAmmWH@Q+zJc>+(p&&ChBQ+^c97;1kSuBP-o(9v!@}+BJdl}AV`rx#n!yS;Ush8d zk+O)CMx?}%@Eo!bdv-pr>+jAXC!&Y)Q4O7IbyJr;hm^n8@vaw)OEq*}cYc)hjys23 zXy+teaEm&|GLblItT7xxecBvkjyx zY3jb6++X#YlRye-JG~PmL*!$S5+aAagS{5DJP1VYO!}5fK?alNZ;%^_bbU8BRZQeT zkO@S#g3Nc-kg<1OhT|z|W`UGBjoWsj?W@?;&}xbt3AdCBX+_C9|5sMGS0%q8OFpWh z6?GA>DEx)BcQtx1aj8cA=Yy0|Rc{4ZNMtrh8Icb`%8C2|l1=R%-1%?#K8{vYpR+*Z zp0J;tB9O(Tc?jffBFjKNBeD%-Gm)b|z&#LGc^OzQknNgtww27u>w~OtI2mHg-_iT zI8xI{PXp;fWDrOpk;x$CL|zB!MLql$MDCgU`t11;u3<^j5@aNi%Rt5vnE*1C$eSSY zjs3npe}I&brfE6OV4cQ}UA`fN&zkaFHLND(ND80+Xs3ckIu2wrk>^1wiF^i9v|G4- z{sEawJv`#$fsO&c~;`d?DRQm=4 z%mt0~ERe=Tia`p9yaG~6q!Oen^{~S~aao?S@EPnS?hLj+ zT=G#39hWawS9k{NUAGyRYUr%;k|^tScLw{AajAx~E~P9vgY~YLjY~C@)r+#^40en= zgDp2M)lgP%%91l!@7iWus?oaayBcQ&ROjX(^8CxMpF)uNq`4ks36bd_ZxLAxvYg0| zAo3d-yf(6K-orWVLt~fU^F7o34S4-E$(*W&#@?64{?1b?CVSU~#-$n^`(%(+)Wf$y zXn8S3n9$^EZh6 z?up;8p1KZu3DS%PnLwJiK?aegW(CghNz)Z%C~2}Fvq|$QNHJ;Zt;b%}Y3%+d6YZml z-O99@qKIT7QW%kRL<$^{id};KOrRdl1d*qle!ji}nM#_&zDb^PdQApoCTSi9k!N;Z zvl?VJY3goBp4oZL`5?u#qV541M`S6;1R}dYvP4>K#C#DM0#ZU`4oE4HZ$Jw62-owW z-{KsJ`g{S1JZtlPz6WF}X_kU~NF--dZt8C$?LZnd47Y;OAdB7FklA?`q%mps`VLz+ zY0d{}Nt$UOD@pSeNSZVaH)DGz%^;Ajq3u%4@=}DTl-{XpdG`E2aAk9*cT>K`5 z-v(>`fHg*%^FZ>P#;*CoXuHUFtJiAsB9e+o<%Q81V?@d$QWlX?M`U)UqR(ksE$czL z5NY-!_DDo-1?fp-8Aygm-7O$B!gV+hWCT_01rT|j>(}AmAX7=xY|L zNO44p90`x_8B~qW*#@|Cw)yCfd{jfn_kQXM&)K}I!njmJ=TZHmtYz+;ZO?5|^P z9Uk!uu7w+gd$t}R`x2P|Qjf^%Ao)agfyi$&@i|+@&8c+g^kNUqC3mQsK225Cu} z`5^K<*Z27=kRsaR4%m*X5+dh-6cf1NB;C>Wo#(~IhSNVPTa*+9? z*>?wONSX^k%1AREM1GgbxAQf~a?&*X6ICV66(FlfGZ#dDbBfnSLA2i}bgRYgH_{O) zh)8}!@*ZdJ-uI8APPU->52)G{|rw!$HOoDFw+A z*$gs`Nd8W&Ln0T0loFW?QnXjNGCu^_NOO_13+Mc_GTVUcBux=W^&`W5!t)@f5!nPX zl*n=apoeajrOrb@ib*pcWG-oTfJ`9GDgR<0MVe6{(@66U$WqeO!d2gF(wqlUPMRqo zrKDK}vX(RraCBKrntmXgN%I8AGSd72vYj-|aO7A?nj7R;f6s3LTO4^8rS>gT}rS zMBXs*`_(47sOnMSTJ0?acecG|B1jF=ybdC7n0U?4AbF%Y49{%j?Fg?q52Tp3ulqp8 z5%~xtOC)c1w7hq?RdxqCkg_I$$XgaZYXwLFY4+Zuda5mv9v~fwOa&<AKF<+E$;x4w_^Of7wlC%RYuhq0#Z(79!TCk;fmTOE^4{i-sm&6d=*GJ zX`TjIP2^{gbwrx(gS{V-n?d9k_jt9W+(^T-_kq|}{aLPjGu}Y#DU;7qv?CwY&{^&k z>OzD*m6Ue|yz5)zQVpF^4~nwhmp!h2s^=yDOEr{rC1uIkyLTOBT&kh0t0+s(-m};< zC7;g5r5eh*nzH2V-Ma=Gmuj@3Ng&&)@(V!ZZ6Ci{)_~+59q#?A?OQ#y7m;Ql`xChU zqydrfAem@RD|^@@NI0j1Y3y>xwJnaK$!9Uzk&kLh1R2v(`s7;%;jEa^PQ*A6QrspkT^kk_RB-;Z&46O{ri09d(XM&+_^KmJ4>0?XQs%X5wbIvLfo_~ zZ--=(ltVluUqZYj9nvr^mveVt56PnLk3z&xAKBgaLVT3@8#11xf14Cz8p)lIB9dK@ z0<{w41hvI;9A&PC6jEj}WI1KthZL)fbv+b+1%|&r7*}h|%EUB@DiO!iwjHj1G(Y1Z z=_G3*Zjv}8Owu(SLdQH6B7R=Wp4Dd{6_oi7@+L{I_9;dc$z70YlD8otH5+0SZO%$D zYAG`XBL12uUsvPWj78LJSThz@A|81g9w$O0ZPWpyAQ=gXlB|VzTRT_XaY%xWurEG# zTl}n=JwNjyshQ5}&ml-_lFprQwIP`a$t0f&@rjfq2ev&R7ECrDN{u#yO=KyAhI0nPre$ zNZyCsL2^oGtW1)Tkf@psF|MVM5M@4rh`$qR-`jLJ7so>xKcre^tQm`Hvl>uyXl4AG z_>_p_*?>p(()=8SWRql`hjT$P9TFpnK+5Tu6GCWKduCuZD02s7KgkZrVUoWfaguAh zU{|Wy5ThuA9H-1Ni1>@BybdGUjCs{;STp8PA|AOHUx}W%fhdlsV%fyegs01c>+x zw)XDc2&tgVPZ04pZTVb;v~yvonYYeGVyt!VXGvU>m=ZCHte$w@p;^5jQbF=IBua9+ z2Um$U&U29q*-N8%0wR7&&^{MmLTV_Jd2x#I4auF51j%kl1BvSryn5cV6NBA^E{Dh=E zKR-cwP{!LQ#ppw_3gRWHgA6CR`ck}CpvEpnu^y63nMQr_YF%Zlu?MvI39H$#G9e}6 zk#EA|Bxt0ML+VMYAx2y0YH5>&^F=ZS;vxwNY3HnpLGox6EiS{ApXP7~q=+&LAd5)$ zLDrBoz8t%NWH@AwnjbOJ6_DMO`5z=mnaqB8kC8I7Af=Sq1vyNal>WH-sf;zxe(hWY z)y!L&fF^!Td`iS9MxvLSW_1N5lcWaXA;}nk`5~DJiPI=vhKQeEwb%3Skb24t9GGG> z>h9d9iyc$A1oc6)KHqmd4U zRFces#7G{8#7Xu+!s*V{`!mEt$DDp8u52`810mUznF<+AvJ!F&$s3S4Bqt%e)oh4y zU2qlFDrNkTO3JK&lv3sZRFjA%VI^V|zoA!$pNm%4;yI2o10jn@WmX}MzJO$^`4OXNKMb$YC^HVyTV<^IacOfH zQnO)Yf=a|AhwwOV8tHyWCQ0MrxI&WjheSvUA=~H(Ux0|8X}9O+8%Pyp&K{9sd_*!4 z5+_*+IZjdo@v5>bLCfGnp>2_!(7J&-VE{)QA$rtet1 zj-$*hNC{=OKq@G646>XuopbOkLzxMXFlAOkVw8CwvV}4&#^HH~GFL+?RK{9k2?)P) z>r<=F%6K*LXyVqyrHP?QVzl-gr%6nSIHy63!bNNB8Av+GA&8sgH%OSoeI3?3&Cd{s z_{|P`Wln<_7dcmN2yzO^YmhXOT8Nt@{d%0!)1A*&!yr8MPGJNG_F79HX95b@g|_WXP5~3I>ua=-%8&)QuiC+_+CSFZEnz)sS zbvPAA7@+fB1}P-j4=Ew}6H-Rf{T8eQk_nJm^DMs&siI8NJbd4rGFL*XDRV!> zPnq42TFU$hDWFW>DcC8Lxf4=InU^39lu1B}DRbei*eM?8PMHn~QKlS{Mwz3KwUjw; zDs~EGZh@4mjJ0|r+6s56RcB=kO%gusSy+>pCQ(fynuIk8DRKVnf%sh){&|x8M!4$8 zuO~55@g+0#^8k0qs+Rcwn&2Pt^CWiFF0)Ez=w}aRX}zi%TfIIpt7L|H&8A-B=Sl3U z6K0jnP_H@EOZ+^^(k50fSAO!pWQKa(Nxj6+lh{?4m{l?(ro|Ww>EUs%u{jX&J6ZOw zTn_0^nO7l0NaB!TBu%Daf06WpY|~a<&}Xe5aXgcd@l#a^MEw4jeLPzs`IPwxB7XnN z&NM(`bp5&DHtcSaTu7YcL5M%YdDW_b1gKXnMEo9^-K*7fTxlsY0J4PS4#--Pry%7d zM<7u(KVml03NT}o83n0Q8EaRHJA8g_3izy9wK9Gs;*lT1;{<4=Z$b)5{(uBYE}4N< z*Tp$MGaykq!u1gGJ9PH^R70vMXbd`U7Aa-8G=NIl5`NVOWf7)7hw@e1)`=jzRY z)T)d%_K-F|KHq=m$E!p66+ z-T3!g>sgImjG`EFf-+T*9x7vv-LK70V$^^4S6qpB721= zAU-<6&miLWM(z1I{XRUOQf4sZ4w4|Gh-5pYgyav1__{;+Kw%ED|roOEMo4CD{uJ(-AhFk7pv9pDQ30l(`@BCdocX z6-mkhJX?{Bf#j>Pi%~oYsin;KkU1)2joqWoPe{#%l?f^lkKA`5?tf^c^C8tF2Ox2h zw#B$2b#tzt$&dsc;bw^VePMh3`~pes<=lh)7vbmqNJ=3cNj`>Tl3e%zcDEY47{x+J zZ^|5jl&Xw1cDFV^0W}*|#;-&?vgbjp4SM8TA>wzO?QuO03D9gDh7^*VR)S+Dxdvif z;GDySkZkI;6C!?B+U}Kr_$kxvA^aQ>Nj_u_$7{rcq|8{z z4wB`Ny(I5JVkGSz!Q-g0i&5MJsiDj!$ZnOf#%^fy<59C=W!y@{BmW?J(Ma1piYp1p zm5?aObV!V3B_!0{xuV{J)X_124-vo3ZO_;lk6|75rd}`zl&b^ir)~K&-aA8R~Tp^%Ae>#qaGUzrHi8 zWQKapqh8__yZjLz!I=@tfmzCJrg0OrvGEqozz}NHJxGL&R@A z+mCY>q?9sYi1@ANWX8D;i|Blbb$F9nhx<__zGQ~Z*S&PU#5%OAj+#|6Lu=zct=As4 z4jV5|_L3RubwBkI>(H(`$E=bW>J_A3VjY&Kb=c3Wk{RkXpL&UPXje@%t7Jyb%YBgL zG|x{##BYw<=i&{>2FiQ|*-Fx4MT+q<$wiReBsW2hYwIvh>+lg|DyeD*MEsVzeLS_0 z7-h~_iBVAI3P_l)#kgR}2NZy2adN{A)KS8RgSNbYEGWGI8>L@cE@-xX=NCU|` zkd}R%SN?w?^=f{^>AAEF&(xHe32EBex_h^-&oMQJmPD0^$JvM~FOBOEB%9=S$Z!(( zYP?b)84AfEnFX0Z@;Jm#vKNw1@&lxRr0p8)Zjvh?L6T_@qo?!itbk&7OAuh^X_AKU^GIv8dQf4Qlf--+XGAYyVIqV&kvDRT!J3EOH)*ZE# ziE9$mB&x)DPrHEb{lqQ}@v5U`^|F@|%_Gj=o z;!9>|><`h{D=+Q+i(U1rStTLq^r-maQ!R>_P! z))yf8G#fP#@mV_dJf}X7>pW%pLd0k3*qJGiI9*AWKx#=|gVd3H3rUc)c>#~y%Xvk* z65=A63GwxIW}bi)(h@ujmmxK3wTMal9%5uU zuWT7FVx*Lr0BK2?6_68@IS5IoOtY8p$Y(h34+larC^Hw*k}@wqJd~+}q*KQIGVZ7; zGaiybnI({H%DfBlP^R%Vyqcg)KS+O-v96-xH+uMfHLUiRl?iDQ)FhyZUlX4uUQIlj zxRr==Ivb<#(;9mUl238~5+L~<5+phA6|6dvQIIgnJV=D39OAjuxxznz_-URSZO1vK z6@CGvkTTan=982_T3zN`tJ@$!k{=-1>I{n6=<+Jw$)U_mkQ~aagp^U{J&2z&%`5Qw zoibiX0c8py5z4#>DWuG|klmCycL&~&qRb7DO3EyQgeY?Wa+oqrcjBi{Il9H-1t$TrHHy$kOQP$m}=rOXmY)61P_=K!RNGEH~mdPtdpkZQ`zg>q&H<&K^iC%gLo-(>g)I})tSz#)*#4m%G?V{qs(?lE@cuBH)XoL zfmbDznGEScnYEArWj=;vQKnTCuL~$M6f&4H3m_$w*$weg<|Je}WqQ7ey9>(XL&B6< z2g#?*XOJzFX|o6K(^6(6q=GUfkf6#~SJCik|Gm<>)RoQ37@8!8YxljH#59R&644~A zNl25RCILa~P=iB}wU)dI6hX5_qVf$XLI^)WF`iA;@u(J&;mu=lcKObGnqq-eg(u*Lw|Xp8v$-h%cF;u`i>s-~Qwm zLA%PgFIgosG|$Va*O`mMi5pd~jb@e1P_Gr#Yt)t3gzT!cx0AhOhI*}}USh58P`wtI zRWd`pR#C4@uAcOPUG=hAB{S5kOzU;4TB~21RWd`pR#PvrR_&^^car0h8S1r$dWp|x zo1w-v*sPKn>h(DFTDS1X4!i0>vr1;D*IMc&*2Y}bYrk0~Gt}z|>Lu2OU3E@na$GV) zy`I#1EmFPam{l@Ey`Iu~*;Q3$mCR7Du-0py>UG}!=(YTBr58 zTlG3*R>=(YTCer8t1{kA9-++0{dqT}o~{5JAmTGW?Q6>+QPt0R@Anr(d^V2GO*`WC;gk+IygLp|25Fg2fA7K4tIp?_mvYbY-2_imE&z|T1LAFq)!-pxxD{jz5=lL*Rxlm>{q>?hbAPLGe{Rq$0Dr3!IP@8ABnnNq&(!@|Aj%PR? zxq*5ufQU~Hv}fa0h@0l{FGwcI6(8e#kt~FGNh%@6WzKWa>Jz*|?C-ovjE0EM?z6iu zg>^qeNhot2kX?{u}=_Kz%+$3#3!_1RRfFwvZKniFS#~|YK5AAvGR)ebzWoAMi zA=v?Wg5)1agv9qb&S{$S_43n@-IVzi;-bvpFR%+Jvl7x#WvqGjYx5jcb7*BEnuL{z zc|L|mHZFIr!wdh1XCji>5Esb~h?~TSV?~h+g&d|)JOUA)tZ2`3404<@ZNE%0PLfQ5 zqzrKGi*iUh$zPC6H9ulpBfi2NK4rp?-jw+T;-Sotudz>6#+pN)HqRk7hgK%2NkEA> zo>h3{1kKMEkb06Xwc8J7tDI_K_@r zd_=MXQb+PPB%o$YjMV!*zRN(FVn`8X-hmhcoqMOnaoo44j5VtsZRR~{R;`R%6PFTk zJR|YQEveUhi1^G;dp2H%xM>c*g=CVP^Mlxx&UqdO@sgB6jDgO~K1dpM{{tdEP1Nq* z<43%sNSR5HJ|s^vr7~)swR2&p zdA2f%q1yAQ5^+3}@yITkpQj+{B%eXtB<+62%#-9m5+utYg*1wT5b?>S_B=QH1^1AY zxeBtHq!{unNfc5+a>}m|H9ulpqak}KvqlJIzJkOk)BOZQWvn@LY4aRYb7*CvnnaX{ z|Hm-24!xtf`BtJo1Bp05<%###A;v`!kH8hHEA>z|t?Rn1l4OallOoIGPvI%n9 z70xR_Eu{Cp3oC+YPE?zOzmb1@H+O7a%OMbhF=%m&F=h@0ebNCwH*kW7;9f8p9fG8^I{ z*$wGUlG1?JaU>%lUXoRi;2`I?z7iRl`OLp@^`o^h1CmRbS0J~LH2EjR*hDf65+?~m zj5f|Y*b|WBlo|RjuKbjF0#Z+zUm)p}x!P!EH1#@H=HrkI%KQXzQD$(XW`>6{Wsr`P z`2o_OGFRenTxU{dB_x|N-$8m)=87iGj2y}=gLo+;{x+HfrM1X+F#;%kiSPCQ2Wcu_%-oq;?=~XiCYtw zCWa=7Z0#9Mlb9w^O(IGRd>s|DG03@hIyG-*l#%2>!X%3!8%Xv*A|!tb8SH%I-l@%u zB03k-Aa2@~Pe7Ja<}l<5lGdj*Gj@|?L#jziA#2rHHt_Wxq?R&gp4!YPr_6Pb1ZCDh zwo&GDNCRa$w`gWWDU%0Dz0$dJ%OO>i`5ux+nVu~%3YD=|U0hq4VRZ(rOh}WUCIL^}nBd5@>{Lp8T4HV-{aBL+=uAlvOx*^~oo8)k?EUX6PO1Obn&? z%qp3oUe8c3@!bl$>T|P7W~f&=^%CD%ThqiEmw~^dCB9^adOb_M#CO*0s%~bL%*guD z5I3FWyCLGU)9t&IFvLTdcOjRN{0X^&Bm;k!DVyX*NUgShyxG=uLmbc&WTLby>mV@_ z@i$>yL!3Kz4J3zp{R9!8?rx7_kgJ(7fiky3#3!8FnG#4IWwt`Zy9ajWeTbX7{{_h; z>CzfAMluTGCAkM;T6V6bK{65IAz2DZknD#XrcwL_5uej;&vOraw)%0(h|iusNwNl#a+UKcdI*wE(yA@4 zqH2D`xJE%TC=-IXDHDTuDAT$fcBRT#b12@6_(<^Ij%C+%m=lzW2iZ_GKs6T=PxC4p7F>rnx8|E zYLX_II6@Nf8951(nUKhE=ge<{cxV)#Ld4(5u;;l&H|%A~cp;-n=0R>Di9qI%`~s<0 z^CQO9>jLab%G?QQ+RnPy5}ECgQp(grTqdov?S^OFau zBzX)HBiRXwlN^PFM>yv>tp`>wjbZ>KLi0RD2xUT$9VBl;_LCfk)R1Ici1(V*{D^Vg z2&tpYGDs$6-i4f?Oyi62UX#jLbLiLRIjrW;%7iotDv|SyM~;ke&d*Fp1<86yl;l&0 zccinbHGYTH7~EOY$49P@@SIveP#gmZ1l!-%z ztBf@pK5fPVYBsElUx|3+&KKjkl14fjl0z~d;wO0t5-0gaJPsYOBm@po$M_45&=f-;T!G&9~L=>w@EnFOgOSqllMv5QfB z1gWJ=t4lG@Dr1e^qs@;;&4!h6D{;PS^jxa*T_f?^g2Pf$L~DGBpMVozGDGjKY?f7M zD&IBQa)$d;yQ;{nk{Nm@XN%VBcGc?zvr1;D*H*2UUG=3|B{S6Pd9ByCO|5aA(Kk6R znW0`UXua&JYs@N{psMm{HFS{yYR>=(YdWm|8pGm$=y~|f?R>=(Y zdYO8OpP#p@jI88r$PD$`ru90c-sS6HR>=(YdWCw4clqq9zGjuoP_OOOOT5dsT)oRT z#;lSV>h&u167TZaRkO`1nUU*w86-iw>19Zq_VNd!>T2in#Rn>ow{n){k9vTEFCM$c&tqo{;`D&!ZvYv-0foH48GFGAkkD zNTQG%NopZ^ByIZRnNeFmVr3@RkMo?qPGc9pg}qQ+SBKzn#Fxy_*x#VBi{HYwtLB(h zG9&BPK>~Dy`yk>kUfE+m1}UUW%K^BDq>T89m=Ha~j)9bs%z=bSRze~qJ0Jm{^SS#A zNRYb!0}+2u%09wQ1JR2zS3sU7nF@K4&2@8I>Kuq;uBfzIlKcR zUY{1)*D=T`BnKd^NKQi1NqSs~S=Gkw(#9U8aZN&1oR0ZXNFB*eNP^@TBrw{!TH0QP zYcCy-7b5<;mOavGkp7fe0U1K_4n(|jV$a5}kO?F`24SAnNX6O7gXB}@2}q5~SZCSL zW+S3z!^(t}IIjs&x+aM0{2XKadLQo_P&kr%?CNkQim|g?vi# z0_0ng&mavXY1w!eLd}?%pW%?yZ08U&E(tB&?{ej{W8Wxjw+A?b8I zdXZcQDI!@5DOcx9jN(g331!aD#g0=M>wJmNrsMCnhtzCX8S##Na>m3X7vOOWJbDxS zxA?~}wn9=#euTJ4dXC3hCAk9Ltuoe(MYUNCs5!JUeocHz#PL*#UNk?gCgQyU zk}(hib8VlC6_7NNYKWVp!%cVc zeK@3+GD{&nDDw##?x9cO;Nh-ng4 zB1Z88dU4g+5b=pg_6ooF4$QpDsM%=o-x>3(`LQxSCC>L--cH3x8{uE_OT6E5mwLZt3VMkz znW6V3-jP*!lSj^yc)!K2T3}Ym47~?Zsr5S2I7N2Acbv^CnW0|$sh4=a#je_AR>=(Y zdY5{M_glj1{gzM6Dw&~P2dI~Lzs0Wl!>p1SdDLgk!Ygq)JN+QyZ#3I$<3>nF%G?L( zLh=lxJIT9{-XwoQHfZPFtInx7pl-9Vzi9R5LZT!k5Ciu&Vs(pu3}Yume7cvt#*RV6 z-$b@YapoLcAt`eeWIV}S$jv01Ao(O;K_c2HJZcoqm03k+P<#T{e?`OnsYs@M?lG!y%QF zSpaD|!nq4}Kn_#pCy4mOKYOQiorh--%1nTCq|7qNamwt6WKzZ`!VyyDQb=#g%z!jK z%etSpuIhg6%I{XY$;!AiF*Hfw=X?2{B(910$zje!l{j~9jCPZFPAOE+DeEx`@g*~~ zo8G70B%V|3s<+K5nW3Hgf!3==J*U)}RWd`pKBQjaImNDOac}YnWrlhkre5MXWu1CX zxzMbV8S3>B^%Bo1cGWnuN@nD|6hXvi>e_2;Jw*IPczdVphlo$&wKKm!#*=iu4_9N7 zYa#h0L5NFRQ4#x`&dR%G&Moq=jF06g68IlY@lP~%^+J!jLcW5>5nnPR|LZse1r9my ze)J-_0pccE3W+OmuBcCBcf8;=8sneC&DJ-zyH}w~e8~*Wd^OGd#&wtfX;(E0CaYwI z&hn?S>J&Vp=(Wq;?J~Qnk69%%RCPpFi4lsc-l`8D#Gg*Ko~;VZDw(0G&r}tPjM1Ox zY__YOHLGNXs%q4@MAfH19UW{}#my?2q1pIcRf!{fWy&9O?W&CV$=Q$@>h*=H(tAxc zt7L|H{ZCbiUhhwDe}&y^vsooG)GMy4^j<%jRWd`pzEo9uuS*sr$0akyMR(m^FlSPM zcyQw@StZu8_`2lA>Ui=9t!t)GnW3t$WtCVL;_I;!w*@YdXW3aLGgMWpRrS)UWQMA~ z(W-iDRWf5-R zDx)}ggfc@_->WJy0Ql2yW~kSX zs!H$GVNr5iGDE$7QdQR8v3uQQR>=(YN~kKmSJ@cPcGZ(+mCR7ppR&riPwlG1W|hoP)nBR#!!{<3scCXVP zN{&lrsF#6BZ_-GdFTIz~tdbe()ksz8y_T6(GDEwdv8=N8>6~VNeVsf)=RTDgs%j#u zoclCQ+dDEtRZX?3Hd>X;P*sXn)mE#L8QQ1KWR-KDwp07`BXc%nM$U1Ihm-r%mMbA* z727fw;wE_>5?A8fr_E(|=RUQ&A48S+k{LRKsj|wsPwlD>i<4C{L#yQ!S%rV8`(C?h zq**01RCTJXa_&>Rs@SZO8LDcbs>BGyKDDc&W|hoPRZCgrT+epZZ)TOu&}^Khs;qr# zS6x<`oDG?wUZ<-nafITUU{}pEt7L|HwNh2q-m$A*H>+fZdbw1U-phC-d4w`Uy;`d( zz1JYKN@i#moFS{6`!sok&V4E~RCT7Ta_-ai+TM{Fs!G$U&eE!6hN{|VRUNb{nW25! zR#rLpX-BnB7n!pmGh&L3T@YMS4O@POh*fOM`Hv>|sVz4_;wt0Zr|o2SYoChOr*`*M zs1jc?LuW8uR$2SB-@4GHcGUs1N@i%aw3k)*m)fUx)jwvH%uv-?vdXzn?W*37C67>M zsH%gi5+iu4*~-awRluy08LH|itDO7PuG(Z)$qda#Csk$bQ@iS0vr1;D*V(E{9HH2! zc2)OKa$GV)z0Ofp*50wJW|~zpL%rOpO79ght7L|Hbyih+uV2k7nW0^9uB@{5>CdSr z41FtEHQ)!oSo$wX2>tt7L|%E|68ueQHt zbD!E(*O^r^L$h(AsI@cP$s=^`Q<=gr ztCAV2>Z4V4(W+#I_UWav%DGRws(t#6IU6z~rr7APBDqg(84VGu*p>$&Zj#p_aV5@u z+E;eB_NjP%YIpw~RpLu#=nQ7bDr=u+UpqEpS9M>Rtdbd8EtknE{7daqyXq#hN@l3) za#`ivr*_qHvr1;Ds-LP7Blvm58yDDBF|$f$sH(rLa_&>Rs@baKxMYTAV}Po%_NiSp z)U1*j>NQYRi6a#I)UH}=R>=(YxpYFG7pJUJUOL%l|+ zDr@i9Rri@yGDE$5s!Hz_HLGNXdW}|9daovHlSe2sveCMj)^PP+;@g*~~ey*2Q)>T6Mp08cC+^murTC2HQue(*Ry=Iln zP_OY?FT3g&vr1;D*9}^)`&F-Vo=T2OW~kSVS}(h5q**01)N6v)Ymw?zWLC)x^_r;l zva7b3RWd`pZqj-^q=(Yx>@UGSDhYC&W6lTFTd975!K6UR>=(Yx<%_{SKVn= z$qeNV<$8++ST z*P2x_L%nXLUSfYOQN0$MRWd`prcy7lzwD~rW|hoPuYBqyK6m9=)$6!fB{S4(8uhB| z7+qplwO*H;4Vj@{w`sl3RK5C`RWd`prcHC)i$$AW~geGtg^22Vr}$n zX8jCIomnL_G_Kh+E^)r>sy-W%vmrCoYmU~dpXxQ=tdbe(btm=8%Sh{PS5=u+GDE%Y zqF&-BQ75R+$!WVWd4w`Uz2;Ic_u@M~v8$$-RWd`p3aOWv!#7p0m(41fpVk(?t6oo-RWd`p z?xkJ>-Z}LhyXph8N@l3nebh^wuPil(&7Mh)OJ=Co{nV?=!>`jGDE$B)a$(5 z>v!F$dOd4a$qeZ0*i=Or@Uj32uh4@G&L!$J^4@3O)$Ppn_RRhT*Nqr7Ke@N0B5+LzI3Q0;KL6Qnc z2}vy^MB>_v-;yBd4+)dxL&`~3Kq4f2AQdD(L82rbx8P@qNrpmVBzHmLBe0Ld8TlsZVJy2BUi$NfBBO;Bb$q&H=j zK>AbWU5J-5jbFgZq)b1^aLUYvtV7Do zfa!swuM+QbU;okXp*5yn^3zq0B%?f-;4W6O?%o(m?~dK082WmZAbDDyrfoieAsig&RnGZ>OVnR_8UDDx`BLzx65 zi!v8fU>8tkGGs7io`7Ui=3|JDGOiui1(X>InLwF^5I<#hL-Hx}8>E0TJ$K^00?JH- z6jEj#BuJSWNHJyF?85J(QDzh*q%zifGCu8h5JKv`0xJ{LB%q036Q3qtO+1>oHF0TT zXp*?ndVj+@=C~#?O`@8Jztq6LB@xynq)AYdfF^!Te42PQ@o3`K#HER$iTKMkJg&GV zF-@YHM3fl#D#40M(DUFn$O)1-q@JYZZd^BRbv}D$K~hQbATE*+B%Ne8#7$BM$s|d8 z4c7z`FC>d(2E^2B{=j1&NXDg;bOL42hGR{U)A?NQOZY zB!!TAl8q2!s&oB(07)hJ7vdu6u?OpiWIV)8@&F{0~{rOa|jgfa&q3Cc8o55JL0884)PGWS3#Df2QUb)0iw`~W#jnXWOcD9ZRDamuWQ zxGD1?|%sv#37 zbH<0bu2NLFgrTznWSiZat7!zr@?Qc9UGAi0!j_YqbUWky5t zDDyC617-F?0+jh15}{1rkMW9?G6BeL$~*@tq0HBiO3HNl1g}^ra~?YmyC4;mc>z*SnQtL`DRb_pSW(wG*Zc%Xj4~@AF3P+Isi92j5v(Z6 zTm`A4%sfaYWnO`tpv;eu-jvDw49`|)JJ3VaRaGoL+-xE6Q90$)(JE zNCstgLh>kc0^*^}g`ZrOXnDalLcRAAnR*rs>yMQIr`7siw?aNIGR+gw#^z zJ4gm)GHUT`MVW~Z4`o(C8YmNk^ry_J-{9Hm9Ou3m1j(k%y^u7@Y=`7fCIN9%rrS}h zD9TKR^q|aINC9O&hGbEu)iJCn$_#}Jrpy9JF=ci`e3Ur}2~nnJ9iFWylMh)-nRSqS z%6tYXr%an~@oc3s*1Hg1?cJ5I_rI^iLz)CN325Ti#HWc@6OSftOoHF0TT zXp-o!&5tHAO`@7aGzn`G(j=%!Koh?vK25xucrB+*ZsA5CJKL^X+M64oT7 zNl=M_uaUU&$L?^x|1%$Qm}DEIn&cQHPIA_FxF;bQ0ZEVqA@w9LLyTF@?nfc1B<;V) zT{p>aNIFRn;wIS!$t0$hGdgeKzt;}A-N>(A8_YDk^{*nDS-q? zc0&qD5|AKC#*er+Cdq|_NJ=4PBvDA1sUxX`BuE+{^(3C3v3_Pd*H0cKTI7^9kZp9ga{y9I?`Qr6siMq9zce#G zBAE)gd%W{*c@1RS4Nf@%37+G;7j6G5j)yYiA*GZlgH%zb8d64?b|)~0l(`}AS~fi!hH&&5*6V9I<5aZ%>XKd`$glLP5UndOiPl=%pf zNtrf(V!u=7I!JHItc1*=%qI{pW!n9P6-Aj`$Z*P(K}sp}DI}LN?HjP7D02fOk1}f@ z8z}P`BtV%Ce`7^aW&)&$GHW5bDf2m`gfgA}!HS~HO_1f3c@lD%GXH~wDRa)hSW%Sm zL$**R3^`7jFCi6_>1;GN>M1h`vX?UJAWbJY*ZkLz7-i0D)Eu8D?9AjrYACY-(vdRX zK0${d3jotTWde`_%0wVrDDyLyRAE z{0nhY=89HWQIrWndQc_`DWFUvS92qaGFL(hDYF1Fm@<1H#gu8%y1C(_%pgdJGR2Sy zl-UbeOPQ22nj86)83HM%%ma`)lzAJnjWW&8Y;FW8a}6X)nG#4TWhx<6lsP4>xlu-$ zp^$3IJPg@DnRg+zlxfkXxe=kvFi3(jrI6i}se&|6=CroWjY`UlfTZ5!+!v2R4pZh3 zB#kny+BG-gloHOeqe)DYs3s9j!kUCM32GA1#IK1@ z6R##7P28HeG%++uT&m5FCNWK-nnaWs_!^6=ajeMs-bD~{m}D!Yn&b#1L6UkFp7ri^ zK28?IM_2yIkPy8G@(3iKGCLu6ko*8yOmbES+&`1#Ksu`D6fv$xAX_N&4kVK@Mn~K~ zQ|59=Z_3Pu?4`_Bh*xE-=P$SR{1sO7Y-K{41T_gL5u-SYUQwFWvpV6ffMg`3iev#K zMzS3eC;48;ea`cB?%AlKk>)_w()n5hsiVve$j>CdLYm#|yh}O%9Go4Je8_M$hhn7V zkW9*a3(2KSH#g1>Wp0DyQRX>_monc&0xDyj3zv4j{A%W{j87A znOaCaWjc4oRe~}TAx-_x)w>2#L79&sF3OyFKCTj!83pM`nNmoMGL?`_%A{oCRkq4l z>(J2Fs#~o(E926{&?M1Edj`=Yrb$$jh$dkr2EMMuD7>`BW9c(6{I6&8X#RrF1Z-5%}FLf@<^V7c+?z#7nlV_Chi!s!h9A$24rkOWBta)P89(m-PL!86e!=i2B7@jc*_9LN@$ zpL-zHv|65m?4`^*kb@*AA;(BMU5a%$$+-^4Knm5&i?KfdaZzR$q?j^4K{`?bXE~~^gqS*CDx*n6ldvWsC1UIsWMQq+xflxxkQ6}* zNj5@)B!?iyB)>ySNZgm<9Zr&=kTQ~4kO;|IAtlb$QVH?W9R38UrPb2ra@?6yW+3Di zk{OW4NLE6kBvp`Y>g6QMiy;YG^AjP1DN_m=LGmV~fFuDaCFyY`&ayhEVt%GV z$|$oQ(t|QzKsHe3tgG??b$l zX?YEvQz&yaWH@E!LrN*L3zAEj-ymfwW37acwh}_>d|8>GCILa z49O*V6yhg&4H6*v0pfYYc}~w6iu+y~>2OG@%lZ4Kg^(i3JOf!oQUh5*(&Ad&fskAS z*`Vf7jB5d88)aUHL@4tIBube+!?2fCM$NN!F1%_Et&B$#w-RwY0X(vo=4Uk|n`95f zPx3vakR)w5RvpOzNQh)AB=)HDTr7iB(MWef(&${oA+?k_Wkhr12a+sE^C`~#bqmBz zvIHH@i!w7HamuWN45myqn%}x^cm46J`sBcyQ6;`) zhUV}gn#0o7qffW1mY7vCL+9&ZS%rV8&p@`T_Lx;NLsg4q6@Erneg@!e=hXjVSN&#I z$qZGM%Btk?MCqCH7u4b0DDiM^by_VQb1mCR7D71T@Y zWxMK6vr1;D*GlTu>)VtCM^rD*xa4fe4E0*Yy$br@Yggr&RWd`p%Cug8s$Q$iDw&~P ztEtzM7u@uMU3I{$k{Rl?hI*a3IGnge?d5;WDw&~Pk5jKvS6&mct8Tn5IU6!Vz1C{I z#;IPfnpHAGy`Io|*;O5{Pxg`->h+}7>w48IXjaJ#^?FL{WmkP;R>=(Y3TwS?R=tMi zCXY~NsMph4FS}}!StT>nYn|3>vg*}%e6p9!P_Ok`FS}}rStT>nYlGHnit6=_StT>n zYopf7uFAY2d4w`Uy*6pRMyp;WW|hoPuV=JgcGWSnN@l26xz=mE>NVuXnYpd4Ft~z~U zvX{(IujjR1V^yyLvr1;D*9%%NyK0|VB{S43qV<}ldUd)fd4w`Uy+fZdR54(6x^XS!aw2@ znC+@*W|hoP)ec!D?uNvd_ylIV>T$D5W~geXtg`NH#HRtScArZ*3Ng9_Cqo$(|9VnQ|5ArhcdGv9aYBp8f`c23-S8BSiOGV zj4JUZGqf*WqkSP>zuQ%Bn^iJHd+>FwSFO6X#LX(1psIzBX-qlvr1-Yp7(0K zI;dVXW|hoPuYFoCyUKN2@(5){9_t`T0nJ7L(wpXaEo2U54nVw=`2!NC_YW_hj`s#h z=0WO6c0&>*e?#g?1{UC$A9KDlc|XKO@+Kq{a%NIy;9X)m=E0ERbj%ANAnL*@#H%vadre`r zRxJr>64WH1L>%+0c;qn6;lB{^7e>YXK=Pg-`wpBhl4X#3I>JvOxirsb&%)0UO?5sO z+zM$!vJKLKqydsaGI}=7pf+|tjbbA*#uDc-AB8j}Ie!kGcS!OfE|O;<=_KDl+$7!a z#B%}33`k_Dv-^u8L!(GQ@@Rf8x(jzQl(`);f@C{n63Oq7B9b90-4iBTqPAzr6YW*Vf1GOt0l zQRa+dJbS8)b(VeFN{FkoY-M7aL^X+M64oT7M9kQ&IGzBV!A+1tlB19!lCF#J3W(%Z zND0X%NQmSpq>QBN1K1ZNQz8CU&K30>BuaBw2PtXoyyx%oAYNNhW(wp>lFg8mY0h2w zJtTu9s{}tKsLqa^BuOd7iXyoN z5+r#DQbKY75+XV65v(YZQIIgnQiy-Gb47guDWExQ_b9I6t(`0CI!G~P${>%B#33(| zockEA5+rv(`m3`e=I0Ga4P{ObVQo-m0;G;In;|)r`2%u-GTBQoQp&7^7`Hjs{5OyS z%Jf=_wMv-b4U5lrnQ6mysNR zOeAT)3gaS~11VN#N6gP5NC{=Sm0?_zSqNE9nJ*w~DU-Dt$4r@JkaEiW0NFyB>@|3f zqs%%;1!ew*L{-K*@9tCo`#Ty@b>6K^M3b;4Ax(ms1eAywyZ&*kC_00SA%!GwK!PN{ zLP|)^UyFGrxe-!E@)#sc@)o3=)$Nbq2*8c6ti0Qz&y2q?$5MLh>l{KS(WQ&I#jOP{t2Q zP$mp1qRf|&2Fi4P8fS+xlOU;?&S&a%kmZ#58j?nt^VZ>fQ6>-KrpyM&7Rr1B=|Pz; z>#++cb1NiEWvmtM(sqGgtyL@I)5NQZM-#UuE=>$g5*&a9Fddatrl>vdGU|9p>GB{S4(KlKvtKigGdvr1;D z*Spk9y#M^1djEN!StT>n>j3o#=I8zDuMnGab+vK6wJ zr1%H zCGkOGB=aDlr=0h_<&X$9Xvmi&^B@V5mmu|O#>D*8K^iD? z&Nh5+qMP$Ra~vdfrt=C}3UR57wY%e3DSQQpsdHgvqMAgMh~wFVM-J2c{0u24ale8Y zBN+v$Ah{0`CD{t8B&mVKNLp;iJwM6ikT}V0LY{V>!L^VynxFlU3_61+Aw4LQ@haXW zCK(U8nWO|#OtKf!QO&%Vv40^U%Ji$i_w^`q7i2AEUV`+d%ny)q%5>X-@9R+}53-Fi zVaRaGdWMwtGbm5X%f^V zpow1-pC(>SJW9mb@#2^hv|8?h)RSz1G+pOhQC~q?lBDm(HGyO#B$K2V;vv}y@scF| zkFB$h(`stl@TSyEg^Bc?hDt>-eNib56-6`=qA)6iFi1+tNC?HCD0(syLNus^WDr7_ zNQ%lJ31RwD5uS5hd+lrOv&;N`&-=&mp2vNzwf5TkUiUpIkX}TNcmr35$RLn>A`gNT z5cv>f6p?+F;Dkn`BS;aEYe0&LJOwg`$U2ZkL~6Z>9Yy3ckTN3UK~@ua6{Lbl1xU`E z^g6dJ#pm~{(@(YwKyv8LEdtp?nfE~UyC!Yz`&QNH79t%%%885t8LPY3?Tt4;Hd7{g z8*50JlR$P+=5~-NlzAVdazT1L)!xB6Q>G6{7G>@OnM;`!APp%~YbmY;WzGR?!R)ch* zOoI<`1u1h8NH5CF0$D_vjUfFg)AA!+LCRbOl24fukTS|_2N_P8PD{hTj1q|A^~LrM%OHl)aqLPH7+$u}g=kX%D@49PYm z%aF*BidJTQ3@KCMo=8h@y>jUZxdWsZkyan$jw3PzWDt=@K=O&K02xLk;}hI*M7n{D zA~F)BkjV2OMMQo8DSI~kRB!w#o)WYV`-60$r}{*Y=-Tu%^G%RUB6~o(6KPk5CjgNf zK_=)9cWd|>$VAHg15!kp6Ib8~K$+V?W>97sNHJwCcFHUP z*`t}{DN$mcB&E8aHDgGzAw`B18d6|Lz9D&rt$Xd!g0+L6WGLTJ_srnV3T9oMuvYj#`K!#D~ zS&-=Z^ws$mWCUgEe2tR>W%_~?Qsxejx|CT0GD$PZ?kzSuJWF>=l8Fqd!0U{mci3`6 z$_yzrq{NV7Ly8P>Gv)RIL-LiRzj^x%v*q6o+^yfd?ZEoDAIZ=+RG*7g1OLC|w*$ed zwKldSL*It2G+jUIH*amdB^m1ag1X!{Z^7#4Ey+;VD(Z6IyuF~`yj|lh$xzpq)aAZ; z3)X|)l8mg&LXf7kqt<}*KPdf8*Iyv*DU-dfYSfj;X&~K+6oB+0G7DrHuHa3y$MW== zx&eKT%mA_~t;Z9I#?#gj$_xb=PMN77<@El(6l4>TZ6FmyntX%P#dGOX!Py|gsB1jP zDC&9vq>wV}K<*-PP&uBWL{0=LCUP^#Y+WC>8jC>YQf51dyT5o1i*U`y(yHd_swVe) zjuQ9E$9#)1(@ckfOeFF+ND+~*L9*ti*Qd^Rcw*BC&j6V~>oWnQj52S4tRb=oWCM{7 z-{a~K83VFd=k8`v3bKbXRW{)JI?W_=FEs0ut*en_vXr=2J`JyvO*6e0qy>=`AUQ-1 z`2kml$Uu;c5$PxY43HvPpS2*_lxe)NYSfa*Fp%~{o&(7x@)yWTox7XGsXyW#q|75A z<(f(6USQTI($z>Z?yMC*pWQ3}gx4vhnReZTt4!p6ka8kFfK(9a{1aB}`SgyuA7lWH z`FoHVv|{al#(heei6EnidkxSuWV5b@o7YZ|Ig}Z&8Q-N)W)aAI%GCQ6 z-=*OFl<&%Xv#RB~hRLdy8RE{^@j9p9<5uIf_&sitzQ>J5yZez0y%($zEBzi9tXbZY z488AsWx6)&d)$ZKk_>fyOVB^l~kOI_|gu0-GC8vGW|OET28j=J1?T(Ek3 zOER)9BSA{(dOZO$hj!GbAZ3*K2V@PAcE4ARz9BLgWD}8@AnvTiJE26csT8~I;L|SggbA!lbApL1P#UQI`HCBTRr_8}S@VDy_=?`)@k^4Z3iF^stUT5mYeCQu| zf2Yi)Anwy&ynkHgd5}ev*#XjrGF^7!GZAGbfDF(~vSPVrRSR_ulT3jj`G({vaU=W) zU4^u&8N2WvN91IXG9q_4LSudlWCN}IUm%+(({VTcbpRq`K=v7#-p`9bY7yD@PkfiF ztKr6c8b~%}rh*huW<5wN%Cz_k-{fc}S+N|ms=2y8nlU6>i5t(gc;#|hpLrk^M1BLw znU}tokJ*DyzBIxsK`Lksp90CF%m$G2h_v_{>qF#HkOCqvf=tx8yD|R-GL|wY|ASMm zW|FyQoAt@kxhI)OiF@TDyiP97^dpcwB31vz9wX8dWFn1c9LOG8jfEh^l-UCE43V}` zMzn;;RUj*gEC4CinYuCW0V$_U5B!O>xs2mI}bJL<)88$@&y1aj*OdUMHVs zTC-Y4G>k}pkOCspKq`o=2U$!bY+OAf%BJzuEvnyho4A-RU+ zC~>oR7G2Sc>7DsINCuJi2V$=hxe26*$lD+tX%_osVP14CP65fI%p{QWh%gnXN)>Tb1 zS%yT0RJ2H*8*UbJ&{a&Uy4{V3NNz3Ml|&|kj{8Z`>;rIlK{eCXq`)a$ZjF%()Z zG42ALyPL(MAgd|!3rHztI%j7@8z^%($a2c80jZ!&%fs->nn~8M*sOD*u4CzOds_WxsF&U&kWxfHapiE9PJbNiK9%K(?J_i|2nPx}e{R@A_gWo-iGWOo{ zxw@*!wa76f+mI|JZWbfaRZgq=HpnI-2Q>D4v|p=T_4RDlB>jxr+Y`7H)wtC1}P!( z8At_@hMjO?q!C^WGL6>oNsum-*$mQ?NT<%YUPKB(1`$~aQlfKrV{U#t_9|t@fGpNb zGWT4wJ~=w~B$KVgz49k`ooHeDK0T-lK9LbQ7bK6!Opp;Yo^p`cv>JzX#ZI8iB_LCY z%maCr$TpBtB0WyXh?eP0-I%9CHeWX14M z+Ur}0EM3DS6B$y0ztx04DU>U5v$zpm`LwEUfD{m^d?I!Nk<&pcXw3J!46Xc1kc^wt zcV+#PGNPJ9E(JM^$Xt;2ME(WYrK{n_+^;*HKa`mbQu&zVMC3Amfb^kE?;hAunn_kH z+pKDtu1}IFHKas|>v|NgTuSTnGe{Yc&OOooN_vM+0vSLfTm!P0*09;hxE7Qd0WzA% z2OtxO)I9~yULqqv8tUBLm_G%XL7Ag^WkfABlgvHKtWUAdJ;@X)aj$#_UMHJo`k8xW zB8T?I`!bO$K}v|c1~P|6Sou`kgS0-UgOpNcI>-k^egRoeAiYfU%c&Ih&RffL<&I8C$a>jfJnW5 z8PQN(A2*BZLB>+%6OaPRv^X;(nn;=3K*myL9mo{QbUX`|W|DQrFO>Is(&p-_CYc;V zvJJ^n;%4yxy7FmNe*_suq)UIyi^u~Y6-0gpDW+L;Jsa;SbS>@!Swxu)Aa4=rc1}jL zipUg@%|!kHnWF3CW^wMhcp6jYRghxJ)E|JS@h$0lXDrBE%B%y)qD;5*FlNmp>x@78 zwb!+XbXAi~MKkjrXGob6H;ZC)MX#oJ`0pTDM7o`iyMW03APs5E>p?ct${#rpPZG+E z2I)%V6Odj+8V$mC1w=-JEY;O;V_pf8PnnzxaOY|!S&cHYVx>CwB;)>7toT&mUilup zPCm`_OORninp}vTNu&Uzg27h$*1`kV3U$ac!~9gJ6|%p4H+_oeXnEv06ii*;3#buKcb(2xQp zZWhrHbiJ0|nP-4x5SamzMdTNdA|hReV(v7Hi6E70rGGp5IY?BPzE2OkBqPcsas$Yr zMBV{uPvp=`@jj^Q6X9nRNG@eQ2dPV$)|cVEk}?xPno{OFkO7qGIt(WV%_QqwVy;EL zu48OpmMa z+|W$2&c)_hWb3LXnJhyhL)^a)8=g^5qAQnHbu&mFk@nZ%E+8@vq@2e54oE{<`KkqY zMp5PrkbQ1V-xrU9)FAR5NL?bwUW=!nu118PyFi*!W*x{V%_OT)WLB(P=bmKBl(<*U zxehy%W;z0-gvhHP6-1)zu`_8rCxNt})wm6$7iHcD=|`mIh>Yk`B4>k)CGr%=1f8iH z^B*7+DRasVxL%Zb7-R}%Hi68bOvjO!J7w+ynWLFx#R|=;x_<$gKMR)_Qfx?(5;u!A z=!zDn@9xGo;$9|lDM%iXc_4FX%-cZP)5;$^3h!T(84L0;k!2w3h#W8)ClDeRgUr{} zaASTMWEW+s+=R~sloCfk)B5xngVP0( zNgyReR)J*E2pf#WGlACdB9Io8c?zTrk!>Jdh;+Ugd!ER>AggumZp`0;^ruXRaroXw zGs)cZ&HCi&+>=bM68Fjz@j9h6)8!y#L~7oGxxbO#bpt^1X*`dB^r6-G8e|k@8W(0n zw-UJ=!lLkQ^dQKnjRdoq+QpjrlZ?0krb>f^4VE zN|3*aWZ#|oqq$6c2-3g+ZWW{pLs#Y}F z`w3ZYNSP8hp0n}Fg|t4iL5hfM2Pq?R@eX35zeM*_LKo$^r668Z7TR_T*oO&@;S&PT^~1# z)>E+Zl$i)JjWXYXjG#={srWqzWgZ48q|8>363rynBFkKhd|lNflV?b-AvsFiEc!fz z=NPT(;~*`F`~i|fq|d_`A(7c2lV}z@K_=3*=sgW{r_4-{=ZWkFd7H=?)A5c^qy%KK zu8*5V%}4N#PnqjMmQiLUNCjn%eH5RYDDyDL9?I+nDc4N0&XKtm*}AGpCd-h>kcz{T zyUESs{24fJ(5gNIQbuGqNLFe3E;#Lfc<;VFeJ@V~nL;bS0VJC;?Pg{~Es0D7X;0)! zkX$0|9>eoTSHq3@UXVVN`59!FW|GyYsIvFBh2=W;BvYouy>icDjG1P7KS&EA>p*gd z9QinQCXvw~189WrfE3gE)R={5IAsQaj3)9d$h}1V0GUms?-O{gwoTrx+$^33DWOa> z8&{Ar13(s2=0%W(l*xP&`L0V`gS;KO(&ZWAlNv6b*Vnd3QxLLf0u2NdnjHfUj zBIkgV5qT0M@2&J+{R3nr&Elk|@m@*SVlv2P%6tv7gUB&+GNNjCr1$*IAlXD#gS6N6 zakFUq49;VenGE9oC7t+m;W9shbfnBl&tivDW(G(P%IpLgpqXTy%c|_X=L>aJlT3jj z`G({vaU<;a96n#rsy+oWjL2Ug1w{JI#nX?-Gaz|1i#;HV=vtidJYJbHPk>xY;>_>BtzCuV!1uiR`3+x3vQBtu=B)pA`k$1NNdy59DdWT@*`wQSd~-jWP;{ic@fs<$v+ zL&;Fr?`qku9^R4+bycY4x)xlr`^GS@0&htsI(p@_&dGUl;sW zyYF3b=#%;{hd%1pP%^agJH<+#^IiTPz9Lw^dP_3YwTrskzli#@cAfA_JO;^7*KX=s z*0A*cU=8+`WT@*;>T-Wu_!at|a=W)ALtTGSSKE^Ze-^A+-jWP;?GY;r`^N36yRMx3 z^c%ABZYM4AmSm{wZ|dqae*U^(eeW&FP}e_V;s5CKFj!R=#cL=Tvi=pTy0`j&_H9|P zj`o&h$a4R}M5TRPm;0%I$|W^|HPBmZ&Z?JyymSm`_vRbaI@0Bla3s$37<295F-Cz5Ol{`1x6W~pAe@TX{D#j``mSkvN zRmDo~N_R@>tf!PbKSIgSyfSEB?vxU&>%1iy>dF)=*}bz`j@j0gX)7*{<)=l1$46Vk2Y9+fjSPQ%*8QL4}U*)N^PkKjvVD^S&XoNMz zO7@0(vJKZK+wZ^Ide$FYuORsH--0xji4Ojoy+Bb=9G+c0cVmezkVhUL3E6 zWT>kycMb1ybFg}NOET0|&vfn5u4}y|8S1J}T{Dloa9*$;^Oj_&s{wVny|GE}<>lUz z40Sc6u6h@={y10_-jWP;HKHzea(GMM%@2MfUPH-H*P+z4tXGS3f_164Btu<|#Y&!0 zH$3(8E!XKjT;wguP?!6+-zx2soKos+Z1PF4e)E=OsOvDX($A=1HCYnROEP2~E>`*( z6|8~Yk_=f*#7dq~lUHniCs@b#O?RIpZfOERsq7T+MZ%Kx_T2hyL zZuHRm>v3;MhPsZVF8ACB)`#Ab40RnP7Cscq>FdXNlNOEBZ$-9xOET2ein^Y7zqop^ z8kfeaBpK>zEmnFT2J19$NrtSW#j1}Hy61HD!h)T_y2)FTA?p~il25)5cRKyjV9oQE zWXN*=T4#KJp-SE>gSFmUlA+aTqn6u;?vrn@>c17QhGeL#ty;G0Y;Q@1y4tCgyf25Y z`@JO@>T0i+?ON_F$xv4ZwOp6`lpVS<-j3%b8R|M#E!)-ITaux!AHF-zh^>%toGStscY1knXQ90%3G45u9HmH725Tjw%c=~o5*gja>y(Jl% z`^l#31le`2Blq37OET1T3U#^fx`TDOwxV_mSm`_H+8vp#$dhXEy+;V zsnq56PZyoncixf=b)80C?t~VsgWro+Lo(FWhq~Om@OAnw+}&G}p{~=Z%e@N+>soI~ zhPuuWD>)6iPpr8*ubJMG40ZLTF87HwSgX7x8S2UtEB!7Uto`4QS3@#n^%E<78VuGc z-jWPiXNr}a2HhvtVBP60$&huH&dcpr_lY%FZ+S~Hv>N@@O5PcRm9Z?Imt^Sqe70EW zcaW3J^I0-9!gIumUo$=p&ec7ChVPOLjqqF=q5H%dtUJ6V8R{B9UF~vjT=cCzH{S4; zWT@*r>YCaAr0T({`awK$$xzq%)aBj_+Uqkj&s&nAu7T8b;-9@A4AvBHNrt)xQJ1^F zM(F+ZrMDzQT^CT7yT5`}>%(|nlA*2(sjK^b#|+-BYnbOP$xzot)OE?B*L4k6k+&p6 zUHR1YMEAkxH?ER=!|oV$cp9h2WuD2vZUBjr$eREQw`>?^s@dzbDU6)hWTzqp9th2o(8S1*ibYrsOw7Va^IW;Ynits;~tmMKA+&ft)MeWACT?;c_tZ7<8fz_?s{K51k3$MhQ@Oh zjmMoyg7v+(Btvt*+I02Q_qby|jk_d6UDr^TJCg)!fVU(=T?MA=1${mjdP_3YbuD$d zGfA+X@s?z$>pJRkd$qgHYlXKYLtWQXmphXLYu~bXH6%k_BdE(gGe_!PZR;(`P}dFA z<(`?r%J-IJsB5HH=`+bGIg-NL}ts60CQBtxq)R;}br60Duxk_??mZWb$j zCh2X?B$A;KjuR`Mb$li%(LLYwvzSYUMtBR2(49$wb%VDgLtTZ`<<2DE>vLnSwn`eYXOdw3=`G1n*WJ|R&Lr#fdUg0BUJc1m*FDtb&LqJa;VsEf*Cgt4 zXOjNf^^&(FLtT@p%biJrm9Z)wp=7A*Ug~ns`TMo&0&hu%y6&SccP0ted~ZpHx{9dF zok_OpKCJd-JVMD(*ZtJx&LqL=;VsEf*8`^OAf4CE-jWP;JxE>dOcJbDyd@d;xQr@5 zvgu6HeKlr*Kh88758_HRo+&gQcP44A_r+9a;YTtwo~bk*cP0te&)$*@bvfCGhO5L`TUr-Btu=(smq;7f>q`%$xzoL)aCZ-A3Cq- zt9V|Lp{_@%%biJr)yZ3up{^O!<(`?R>R!FpTaux!|52BFW(MnNZ%Kx_W{QYDY4>_ z#AlKTy62brF3HdcpQaJIGfA*2uZv$zGSoGPy4;!MV|{LP_m*U+>lx~DXOdt|^_FC) z>sjh@-;-qPGjpT2Btu=#QJ4FkBv@_N$Mcd5b(#J4o|j~(>t*V4XOdtI@s?z$Yd&?kGsy|s^^CV9LtP7~%biJr zb>O%02qi;Z3#rRJ=f`N*2yaP-x?Z6!cP0te*WQv0buFSU_r83OK6_96E*_y|sOweg zYTI$#$Y9OzmSm{wHR^KbpS8N@|L~S%sOxp=a_665b^Jaap=79QvFX~QU8B7v8R~k2 zy4?9ESg(6aGVakF?Ez^)=byeC@ONVT=lN#|jmNF?F?tU^4$J*WhQ{+IjmMpTg7t^D zBtvsAHC_MeH9hr*xJxqB^%ixx^G~qGcuO+W^|tA{L*EOY^Oj_&>mBNH=bvD$^_FC) zYbkZP`{Hk%SB;JFyd*!RL%%?mL;o^?hfnwww88R}X`UGDr7tZTd_8R}Y3UGDsot6dAc zB^m1ahPvGOCs+-BjYlXM>MEx$_ng03y9&J}8S46$y4?9ESU-A8GSu}Qb-6RidfkVo z{}zu>GSu}wb-6Q1uH0^z%Dg2R>iU7Y+?ga;2mKz8P%`du8Jz`^LuZnw zLF)eJnPel4$DK(!=za00e`U$gcz&euxHCzx@+#u-NQUOV$#m7y_qZp$B^m1aiMrgG zBv@a1OET2;v*~(3pU)Xv;t@)Qx_+T9cP0r|XKzV{x;9gn+p7oZysq??WT@*`>T+k2 zU_I$A$xzpC)a9O;=j;Ai?=8tt*YDKjo|(ZqWNW+{lA*2&vC?Of+B&b(yd@dx+Cp9K zOcJa~-jWP;Z51nhCJEMZZ%Kx%ZDOU*B*8jxTRbnxkhNW`^qC}B1H2^}vUcda+&$*b zB*A*rTauyG_(QGaOcJbLyd@brlk5~LeI}`E&Lond5$+Nzo^^aCnWB5X^Y)lahDNxX zM(EBY!5ZT&$xzpy)aA}3U+8n=b#F@CSq*S^%{&LjuxndFe2@dzbDU6rWIok@aqzPBVp zU6rZJok>2`>owC`lA*5ssLP#6g0<0GlA*3D)aA}3Cu>*hUGcmmLtRy=%biJrHPu^^ zp{@+-a?knOv}=#IBtu=9)aA}3!MbF3JVMD(S2gN#XOa!N4?p&nWT>k;b-6Q1u(JM) zyCg$hHB47!o!6P(k_>h2PhIXz60E7-l8k#?Mqh$-q%+A;f8lQd{m(PW0W=%y(Jl%dzR^Hpzm=l_rzV2p{|mhwU zPx6*zsOwd{Btu40Y8PD}5#j zR-U&cLskQ^(r1!jJ?JgTkkwGE^qC}BYrQ2IvKr~U+&$sWB*AL@Z#*x_&}tm2R&pi@ z)-~Re44p|DiP-i zNxszQMmP75`Qk@1)YXK#+?ga;4|q#5)YX)_-0z8w)Mw@fZ%Kx_no-yFo8Gx2SZ(}& zy)GH*I)b{~{gtox*UjFN40Sc9E_Z(g>m6@NhPql%mphaEu4|aFZ@h+*p{|zH<<2C* z>gO%VP}h;v<<2CB=$T}?wYwB`m zl3-ooEy+;V(bVP6B)zojMQ=%lx{jePcP0r|y~^T+k2 zVAZPt|Qj8{W4)YXl;+%q#+H+V}j)Rik% z`b?4$=H)HPP}hmn<<2C*+T$(BP}fOfx$(PqV)u7U1*=E3cwUkrtGig~GfA-S^Oj`D z>LFJ8OcJbeZ%Kx%o;ojglesfVu#T!8k5DqS8YiojoJoRpo3|uGXOdIIN}oxpnKOxG zXoS7QN}ow4>7M_}cS(ju*qcV^&LqKVQX`(1WT@*@>T+k275dz`(p!?DuG6T?ok@cA zp0^}JU45v_eNWOiQH zi@MyIBv@y9OET2epSs+cN=ac+?ga;zj;eC)O8MZxiiU0+SN5Ho|j~( z>s;z`XOdto@Rnq#YXEh*=lm_&)v9LPB^l~EkGkBMBv?;+OET1TK6SY>NxAOBMhC@R zlA*4F)aA}3!5Zc*$xzoI(-rBwUhfSLS61m60C2% zB^m0vl)BtA^Bmo)hu4nhB^m0vjJn)2GgudUOET0oOsw>o9~1!ASoBt^RC-}7CPp%Gq7 zBXnnyU>#U5o|j~(>pJRkXOhqLxpA(yBtu=-QXUjy|1THr0oP}hyrf?p)PkO3DyY><9SJjy2etMJCl5@ z*XwR?Nrt*^rY?6T3D#HMk_>f?qb_$Q>7iXM8^t4(40YW?UG7X0tjE128R{yeF87?j zRlDjQ8h1&Cx^AT|cP0te{oaxcb&aPkcP9Bx_hDw^xJxqBbsKfLGfA-edrLCZHNkZ4 ztMi)aEy+;V?bPMYB*FU0Tas~)%cxy;W|T)~k~=|q{pXqF4jPX;lXTVl;wxD0M=~^? zJ83-bOcJb|!(vM^H1~<7D@WhsZugdCsOv83a%YlYz2Ys&P}kk2YnDErH+f4k)O8Pa zxid+y>K`7@OET0oiMrfgJzD2=nztlFU6ZNHok@aqySF4mUH4L#duCptdv%GoBtu>I zQI~sW25X16Btuv?ZUhPq}^mphX* z&@;&nZ%Kx_o}eywCJ9#O7V*3!LtV3}%biKS)ay0QTaux!C#lPwNrLsBwiQ{n!4PXBv?0iOET0ohq~Nz{vPdG>n+Jp*E7`R&LqL=d1O37$xzp` z)aA}3n{^*P?=8tt*K^e6&LqLA@Rnq#Yp&_4q4R2cR6Ih-P}lR+<<2C*y1`qLagWRB z6_5dRCaK;kGwT1JXOb6aJnl@=P49~fVYwg4(0Jz2c-)yJSo6Il8Jc^E={io|<1$;v zU6P@$7pcpgNrH8}wT+k2U{!caGSu~&SZ>zInIu>_$Hc248M0m%D}5#j*7e?!3|WiCN}oxBwb)yd zA?ppDm%At2nIu^MdP_328cWnl&LqL=n-kAVGIS<+Q>^rvqF{tXOc#GCK>H5$xzov)aA}3!CK-i$xzpF z>T+k2HF~}FZ6A+NGSu}kb-6Q1uuk=sWT@*C>T+k2Jng#QTaux!PpQkDNrJW8Taux! zGU{^A`N`Uq-yxotWTT+k2-*g|gIX3Q+40U}@UG7X0tP$Rl z40WwET?gpA7J5rE)b#~*xid+yc6&=Q?r|9%e_UpiPiK;c9r@2Q$toI;JCk(M`(i6B z_ahk^&zCeFcP0r|UdPyy49$JD={j8BM9p2eI{ud=H)HPP}jHA<<2C*TI(&zP}g^2x$!4wl3+DG zKAx9k$ogKa^qC}B*LX`ZWNi>DeI^N3skbCU)(<)_cTc!8NwBJSiAN|IT8)irC1;Xg z4e^#_=uGmXSm`s#5#~%H85-dxvC?Of8M^0R^Iejm5&lFYbZ3%aRq7hgOET2;Gj+K$ z$r^oboaQabP}eWi<<2C*ddgdpp{~u;<-R9rrO(Xhgm{FKp{`%4%Y9E0th2o(8S46t zy4?LWRPV26yd@dx`klJm{S~a8-jWP;RZy2ZlWfs7?9nZrmt?4G3w60ONwB7ROET28 zmAc%Sq>i3Re(;uLsB0T_xid+y+UCY1lnixkr!IFU`CPBpIB!XYx^_^PJCg+K6K_d| zy8fUpcP2SiyBeJsk5DqywUfHsnIu?Ky(Jmy+C^RNIe({i)i^2ck_>h2rY?6T3D)i2 zk_>hINnP$tvPt*h-`?40ZiYUG7X0taaX! zjC)*0&3j}uRa*anrpe8R~NXKu-8qa@?6DSf6`KGSs!N>6)p}=Nc!+Ba{qvRiZ9;CJENb z-jWP;Ri-YtSDWj+Zt<36sB1s!a%YlYz2z;*P*)Y|a?i|5b+7JwN<1&gP*+vza?i|Q zo#-veP*;Xn=`+buVP4*n40UBvmphXLYnitsLtWLxa^p|VB*Dt+70*jDWK|a{eI^Ok zKyOKgtQuma&m_T`>n+KUwZG2G-4pIi60Duxk_@fJ0cs^@l3?}i9nVWLbS61atn`_r zwKk)b-DX%nBHGgy(Jmy zszY7w{tDLj-jWP;)uk?XCfTNInA0a7p=7A59(B1hNw9A9mSm`_K6SY>Nj*K2tnij( zsH*{Wxid+y4m~{{p=7A5A$7Sk$rpOPuJD#*sH+imxid+yUh|e@sOwPba%Yl0+Ld`m zJVMD(S7YjOXOdvu>MhAoS2lII=losTwaZ(Qp{~QI%biJrb#32xgp#4I!>P-iNq*LS z_>;FJLtRa%%biJr)jluok_>e2$viVN$|7bJc zSAG|ykjAqGq=?4T@T|;e8f8ued7Q{NkcC9%f-EQUBS^9SJLzs-hxN~lR#RpW$Xv=i z1hRoLD?k=erux~LQ3YjA23e|^leg+>kOuN|m@-6rwAK zR&_2&MK=#l~7lO>7>-7ytt%>P>YrguqnNcGm zT|wFsxg4Yyky#*Rx`uA1UxD*Ofy|++JPf3WGBZG?6IliF29aumGNW=Lr-4N6qUa{M3*7qL1G1Si z%Ry>Trp5)C(JsoI4pNsglR+xqmEMP+fHbAd0T<#qMwz}KIh45%q#K)O&S>!Qr4 z1!eky^wLbSTZ+t%D$u==WbzHkGbGoL97D1V$x`Ch`F@PBfUf)dAft%<4N^#?T|TZm zk;_4*5qS)xn8+6(bBI*GI5R3Caw5ngA~%AR5_ukE8IcVjWkebb#t1)1@8|v??dj@F z1es6!`E`&!l-UAuHj&mtaDNdQ00ZUVWF$crG$i0lAa zuKUNWe8;OYqm`5y3$mIrZ-SIlrqb0o`%&g(kj<328>E6VAA{_oOwDU>YNAYkkjnR@ z@2_bf8ST^WgI|MWQKoSLPF9o|4APJ?Pl04p<|mLAlxcl!X4Hx@1t9GyGasZQWp;w( zQl`swnNbhQ6oT}j%)20Ynn~`{0&_3tXYYOI<{6S}NRA=dhGZEM8B)=B@Ah&-$_yzr zq(n)CpUmrV1?et07NjnbAs`Kj6oF(Dc>|;ck)J_wh}0Q@>qVpoNG_3ULGp+^=E$e% z`}9+g0dxib1X)h^Y112Ua-d85Vbx({33LGK|RUAO%FW zgN!25`X*cpBA0>`5h(^KD@*UpwID5M4G$iJXE^Q5-XOV@84uE%$Xg&oi2MmMmPqc{ z%xJM*9XI#eK_*h>Bamg3$+|f+nnIa#Kvq)bQIKNFd<#-enIp#GTL{Vw2idHdWGCdA z-IA@>E6HRT5*bo)sCoJ+adUqW?d5bWHh^p*Qs)-D4-z>QWEYW}L82AuYgz)5MPvg= z-e>7d{X(37Xf;j)*+AEQ3`iMeUI1A`{0-OKcPC7Dt~N(?DB zq)3UI`&6{&)3taPWEhbhAO%F4O~9Q(H6XocovYq~-Fry#uWPxvw+G3m%utZai97=GKO!H3EGJU+PMoat z3c6SB4zijug&?_HqW`*@vvn(03G=+$(Rz>lD*WTRx0$=ZNHkln|K)lCwI!s-J<3 zq7hb|hG!J5&+#Bdl(`OMI+1xGvx#g3DIs#?bX?P8(`V7EKo(QxMUWbrN#BA zEYQ_RGWkl}`fSJRsb!CRyilb1m|8<&#XVAvuO*D{-?Ji>^Xi)t5jf64?YYjmV+J*z-jCgOm`t z8)OlYQjk(2J3(^4PG8fU$8i_XY77M#R6G67_^>0C`55F9BAK)B8x11eKxPve57JB5 z*{#M>kP^yNeFD!P%Jc?VOqod_`IPwtWEo{@&c+joGG~FTq|8GgqbRc)q?|Ghp2WN~ zlU(;Qvp2HzDkqu9kcx)MIWx(W8&YOSsS>wh7hpVjbOj#<89-zO$S@++pTbOu^aLp+ zatp{rBCmoJ5!nuswJyDX+B}W#KxoA-2N_m7y?=^9D&L>JyT1b2pGf^VxO0j02kArP zQII0N7H&1Z1sOn@7SG@}VwAZWWGH18fXt!HUmyjPIq6xPqbM^GWGrPq1u3P>Ad(b9lgK$BxkMfW ziPoog!e<~;Xzn$h$2Y*W(>tLj$Xv?Y3bKI6J0R~8`4?n0kzOx==&HJT6@hG^%o;~1 zbJ#qb4=HnnBbrH8zQkOeJYB;ilWRzhlJuv@R`hAh{n_sa^i$+ZXm>x7p-)?_>C=|` z6d9~9yd@d>WOuab+Nz%-cX~@Q)O8GXxlfV7s$UY1P%_k&LtXAu>;-pSH6Ofr#@^qta{ zc9i?mvyJsn1^XCZEw07Y1c?^Nrt*Qn66+g@|I*| zth+%9=nA%4gmorz5y)5~4}cUBc^9OJ$ZkizNk4m!c@>1NPCiHp?TsRkIh1(^w&#XqFS&b5!*K@F<^7L-m3X(yj)0=oA5xEtlh{#7Ey=XjHr8srd zO#6f6Q|3{S%ZY3P8A0Tjx3K4l+y*j8XX>ubDv(K(Ir43sGc}X!jRLb8#kv|vrbvl< z<(u(3#Wd4bLFN$I0Wz0Jn|H7`h+G9yO5|BbzD=+EPaxB17EPAoQwgo%AdnKuOapn9 z$X6hr5UKqxPCG;{0vVy}<7WCaNCjmoKnf|-@jbi`Qs!2WNtF2jBx6eYK0WAt%u6%L zI_H~fQK&1QWC{$)HzZGqo5lI)Dx*~`1}P`6On}hIcC(^9Yb~TKQK%T2bb2 zkPbw8e~=j+Pvkz39z?zanQc}h&#Xo+&8z)~I9*Hp+5o8x-PWTK@3Cc_bsXR4(5B>^LPMIE`AzVB9L4n%R!=z>G4!sg>Us~eY%5Gu9N=6ItHXYWfp>TC9)IbQX*}?#JeYv zn?QEy%DeS>3uFdmGFPL$WBSyc2Qr5;(?GH)^9{&+$~0etc~RzSkW$Jl1ZhE;zd@E$ zrsr3fJ7p$;tftILkX*_%_!{p-loENe$~L>MO!rTcDK(_TkYYoM3@J3Ez>s_; z?&>^?F=x?k*$7gX$l+_TSBYE*(u&9+!z)Q2LqB6{IGSTR?J%ybh8_r1CfT6sgzCtwtY^L6n&aGL$mwK!#D~h;n=a zqRiDGBPjC<$XLq!3sOj#Uf<$9^M4Qsrz1sJcoEgdcD=rpyG8J(T$fWD#X*Zp2=FIK5W~fGpKaa&;ne z1uN>A_Z>sZ3@KIOW-$|8Wwfd*Kvol}{3EU)k&YnwKc%nE)gZNK7LS3nplk6dNGrLiC<~ooXnn~tf zX4a=bS0l;fD{n|Z zX{MKh3?T9fNIsGMw&NX^$XOsGh&&EbNaS~rNkqEtz&<202_*m5^vbUT=}&WS`3FWw zD?b8cIAz`jxq(RSop@g+G8AM!kvBndbXDE@)Z2xPr+ncF~C zQ)WF#f68?H6YqMIc@QL@GQWaUP^Q;kn7d|@t6XAsLQcKC->tF@$ucA|q@wQLuT!qX zt=KG#r+}{Dk07InwAzCwEs<+M3W+QPDI&7Rk>ApLqsQNPf2Y;B6J#jujSoO-O;6vs z2mFKo-kivJAZ>}v1nEVj0%TOD^i%7^e{lsV^B~9s%4`71r_8ZYwWx?PcY+M3%o>mx znn|u{vAL#Yy7Eb;)Q}QGij}zQb>u$PqGDRr%R%N4nFBJH$d4c;M2^_ETC|ABB_O3l zW`mRw*#MGPk-nx!RH_z@qSY7zQb5=AF_0q4tOuD+q)BCrhsa=%l|)_unbRqKO?QKo zQ>N#B)uQ>7DFWF{nYAFLlxbe2TC|HY*Mlsl%o32wkEC~IwW`&k)s#6KB#SbSgKW@D za@~u}-YC+moMZ|ODKI48kUT?jmAL)08RN;PE7&2US~QHvIFJG&?}HQ)shL?V%H5LQ z2^WAgq`A)q8B06iXOQ-kX;-aU)RoA1kkg5L1d>nWu3%yl3=DDxgj zL&`L$h3lo6#DkW zT>>(SGD|^*QReV^*sGMe6=Z~Fl9kUlSEpFlFv%1dQm7>TrwERv-|V|TZ!<;zyv;Xg zcR!M$-|Qbpzu9+x-X>Vp>c^I3=uZ)JG+o>E&)c;3mSm`_6Lq;iZxgJGyd@dx>P%hk z&)Y20KW}rFwNm>6s#$e6ymSo84Cf1Lzj&@^tW9((Y`pjFBAuCrb|DPL0bJs^RgSBs?cwUkr z>qMOws^sru3|1>|NrqPAB(;*ik1<$7yd@dx>aLdUn(i&hP*)GNTvy*KU)~nF%Dg2R z>guVM?W%HUyc&|Bu9MZWU0uB;8R|MkE!%aywZ_LR zYMvdhhGeKKPc7SZp|>PMUH#OuT~oXz8R|MyE!Q=3+`?gD&wu1C$xzo>YT2%R4vXg{ z8S3hN-a)+x3*UBtu>2s%5*@c}p_XH9#%fb;#lIYDk8< z&Qr^F_3@TusOx;SY}W*DNrt)xs%5)g_m*U+Ymi#DYn!(uLtPiBWxI}S60e42sOv(t zY}a6KNrt*EQpbhJl+qKkNlA*3E)UsWByd@dxx>7CM)$WLRH6%k_!_~4~S9(h_)OD3wwriHRBtul(FeSIy?}yd*g-s+R5Ag-u9odugg@XP}fAY zY}d`+k_>g-rIziQ?=8tt*WGH_uFc+(40YY3mhEcNI$jOQP}d~2Y}Y_t3~N*N5Jc40YY7mhIZ-=y+a|p{^peY*#05Nrt-aSIc%4cuO+W^?+KoYmT=h zLtPK5WxKxdmSm`Fidwd-&N1<7NQSzms%5**@Rnq#>mjvl*B#!H40S!MmhF1OTaux! zX=>T79o~`*bxl{xcD2fhS3@$?^@v)wYpAy*LtT%mWxF2rmSm`FhFZ4kb8kt8{w$LDZUmAcYo@UdHI`(^ddygjjU^efij9?REXk1dxUmj1mSo79Wvs)E zB^k1wFjf;|NrtT1#%gLT$&mG=v6>l6GGskvtRsvi8M2->R&!%XhO9ZpYGEwNkoAnQ zS{h3-WIb!FBaI~)vYs>6QO1%CS#yom%2<*i>v?0fHkM?_dcjyn8%r`|%`?_9#*z$K zCC17zmSo6!(O7MaB^k0_GFDq-NrtSKjn&Rrk|ArpvDzC;GGr|vdxtZ!F1>wb)o)j3pVe-Y`~I zV@ZarCB{0zSdt;@O=ER4mSo5(HCC>%BtzC)#yZhhk|FDDW1VCy$&mGqvAP>eGGr|^ zRu5xIhOBpu)zes#A?rP3oop=0koCT?PBE5b$XaHsUdECPSsxgyx3MHc)`!MA)mW0D z^WjHgWw@gme%x>WYv^yutF?*GqLQKW;c~I?f8_6MJfdjl_+Yj5mSm{wW3g(%a%aRh zCSUgJFZw49&hwUJsOuBzI)CTxykOnwEy+;Vr(&glD;=!)-jWPiWn!g&D;=y2-jWPi zE5u5ED}CKnQfysg~^;?Jdbr*J`zF*8*=zhPu|M zWxKX|OET2;m0Gsz==SkyNQSz;R?Bu>=`G1n*IKn~*VEpT40Wwj%XV$>mSm`Fy;`;_ zyF)xL$xzofYT2#}yd@dxDp$*PP4|{$sOwv`Y}YDpNrt+F0UCX>B8S2`omhGy1Ts$wyP}h%Y*{+kkB^m14 zq?YZv#aoi0uAkJhU2k|xGSv06TDI#?Z%Kx_eo@PIb?g|ghGeK~vs$+625(7*x_(v5 zcD?K^$xzpCYT2#|Z%Kx_epkzO9o;FOmt?4`LM_{MrMDzQU0c+$UC($+GSszIE!(xx zTaux!ZED%Brk&$?Nrt+%t7W?`_LgL*Ylm94>oIRhhPwVx%XY2xmSm`Fr&_kF{_*j= zBtu=h)UsXYdP_3YwOcLQHPu^^p{_sGvR$8fOET2;ms++ft4lmD$xzoGwQSev-jWP; z{jHYmy2o3Rp{{?_vR%u(B^m1aS1sFBrE5Gd$xv6czxx}`p~{c@mz~4ALw9dUhPw7q z%XZ!BEy+;VzG~TdE%BCQsH>7%wrh{KBtu=5)v{fkPKZ}SGSs!7TDI#(Z%Kx_s;FhV z=6g#r)Kyh2+qKnOlA*2)wQSch-QsykhPpDb;(1Aix(-pxcJ=j^WT>mQ zTDEJFwMhAoS3|XI*Wccf40Sb9%XS^#BVG;3P}iYq*{)IEk_>e<9SJj zx{grGb`ABGWT>mTTDEJJwvz ztRBXyWGu;$)zes&jU^efPBzwl#*z$Krx>e>u_QxQFJo0TmSo84ZLADqNrtRbjg@IE z$&huLv8owMGGz5JR&`@ZhOE<#Rl``4A?pld?Qbl}kk!{%2N+8-WaSy_Kx0XUtbWGI zGL~e>I@4G+jU^ef&N9|P#*z$K{f%|7u_Qy**~Y46EXj~{jCyVGKxKj}$^&W9I> z<-VPB=R^0K;$RheOET1Tp;+nP6bEabwMPj9YQyi>sy(Jm4^2JL3rZ`yjddDYN z$&hui&dbf;ed`#k)4U}aT8+VKxz%vLDGt`%-jWP;4N=Q>E%TOSsB5TNuFL(moT01g zsqwrdLtU4sWxINMOET1Tsam#cg100?U6-k4yWaMeWTk^E!(xjTaux!Yt^z{?M{#9B^m0vPA%8vep4Lw{B_=v40T zmSm`Fgj%-icW+6Cx^7U*cC|huo|j~(YouDX>nd+ahPrN4%XZE6mSm`Flv=iHv$rHe zU8B{qT}Sne=Or2Hx=AhDb)~l?LtSIkvR%)5OET0oRxR80v$rHeT{o*`yISVO^O6j8 zjZ@2ZUG6Q(P}ePL*{(U>k_>efs%5)2c}p_Xb*oyot9ie8UXr1%@oL$w%e*BS>bgxW z+x3*UBtu;j)UsV0y(Jmyx?L^X)$Gi8UXr1%JJhmWmv~Du)ODv?wrjSxBtu;j)v{e1 zyd@dxx=StF)#R*rUXr1%yVbH?L%by!>bgfQ+cnEulA*3iYT2&uyd@dxnyi-XI;?*@ zFUe5Xy=vL6i@hZo>bg%Y+g0o>$xv63TDGg)Taux!`_-~tjn9tfB^l~^KrP#Kk+&p6 zT@R{dyJmVzGSoFiE!(x;Taux!scPA-M(4!yk_>e{q?YZvz*~}`u7}mKT{FBT8S0v* zmhD>WEy+;VbhT_(gLC6~Nrt)}QOkA>^p<3(>(T#H_C8=%P4D~v?hlQo7=#cGLQy(1 zb7sytgoz>)lTw3ZqEw^?VNeW05e8vU43ZEAMHB`hNd|)u2B9}$5JLFfYwh(sd*9Yv zeZIfBuD5HyU+2Epv(~eowe~vuoMzPJ$GXG0tcGIEQkNg=Bj>UjiZxqZeyknK!f{y* z#kxyfeykIm%W5dr-RkmV)i{^cP^?;Y`LR|xm(@_Ld(`E}+Tnz7TvkJ|?p2o`YnXFc z4aK@oU4E=voy%${)*N;DvEFwstD#u;tILnI-HGA2tcGIERhJ*@c;~VjiuHiH{8%%c z%W5drJazf88lB5(DAt4O@?&j#QaCQFp;!;8%a1kKxvYj_J*+N2)=kc3H56;Uy8Kx0 zIG5E>tVh)4$J%;$I4-N9SdXg9k2T1-tcGGerY=8LwR2ew#i~=6AFIK+tcGGet}Z{; zRwsw!vKor@gu47#$2ynQP^>4_<;S|-xvYj_El`&qYo&8p4aHihEtY_5a$I3k=9GBHltY_8b$2!`%tcGGO zR+k^^8t1YainT;teyrurWi|Ay@;U2@e^UIM_vw<=kn4HR^}Oe@8gjkhxnA&GRzt3O z&sFcatcF}Kdaf5em(`H#CC~Me=dv1dz3jPO_FPs&uBD!9spqm9a=qfYUh!O3L#|gn z*Q=h(YRL7P=X%X^Sq-_Cd9G!i%WBB=y61Y`b6E|!mV2({p37>;^@itq!*f{;xmI|t z6`sp#$n~b@ded`R4Y^i&u9cq4YRL7L=X%R?Sq-@wJXeF~vKn%|?YZ9eTvkJ_cRbfS zp37>;^{(f7*K=77xf(rJqvx_3a=qud-t$~mL$3Ed*ZZE!YRL70=lZ~NSq-^Xd9GES z%WBB=q38P0b6E|!KJr{2c`mCV*Tl4pqHRSr#bA9T$ ztcF~ld9Keqm(`G~$#XS%E~_Ee=br0x&t)~_`oeR4;km4aTwi*wFFlvlkZXwD{pe^MN|N}bDU z$n}Hv(40kYd-v)aBiH%PWi{mb(YkWYF!0lVv$vNvDUtP z!ex&+&k?Tk2vKn#) zTmJXoe?_jHPYZKsHI$83>XP*pxrR8G)ljU~>XMp|Tr-@@YA9A4bxF-fu2-DPYA9A) zb@{RWb}p--SevQKk5xP>%!bua&2Mg9@%8njSMydwu6CYlf#vW? zSG?xuYt2t^4a^2xvRztCJtt(#hk?R%bvKn%A&>l+7N3LI-%WBBg z(YoR_AG!7!9nP24kZVgFm()h&I^MaghO)7hx}@eKSG99l4aLe+mmh1nb6E|=+FD&w zhf%D}P7lXrH56+bb@{OlaW1Q&Slg=0k9CoASq;@;C+mvW{G(nSS`E3j^IVU4E~_Ee z_MWTGb6E{#V+ZS!dy4sMDJ%7!vcQeYYA73>txIa&{I!(G^@(#?4aM5gx@3KszfLm9 zekah(!)9Z``LY^{wG+kK@#LOwMy~yw%W5dr&ej!QUy-ZSxvYj^+Qqu?r-ba!;0?Epj9hb_%WBBAtB%Xq%$Y56t#mG{p=|7?E?HlZtIgOj zhgL(ik+80K9S-)sjcGO1!`-b*YQy}Fpe?mF(k_&& z#`H7iKX8oJ#^ugsH598W#hU*0^wyEx!!UvtD$V=W}lB-=Q@|wP^=<~W%i9cjWyf3tcGG0Q!KM@ zM6MT|%W5cAnqrx8y`!<#IG5E>tPI668j6*rSl7M1WV^@BA6>UUMb2e4 z6zc$r_2u9(M@Ftu&Sfp<(0_4WNzqpumE>+2@xvKorjn_}%ybI2Ky>qY0X8j5w0 zb(xWvyWYSH3T}^FKRTDykn3RUGL>t7CNxfM6S)#+g*mhuavfq_Qiq3a*6Y;2H;5V;~T$7#4YADt~ie+l! zN!>S=IhWN?tYava**7BBAI@bp6zf>)itiicx^HwpI~tG_D~uuI%LtK8aih zofBrmYADw66wA!lml|ujb6E|=8cMOuT8><=IG5E>tYKcP6LftA=Y~DB8j4j$vCLYI zTp8!G8j5uS#WHL8K8-cexvYj_ok+3H*)Zsr$aSA{Sq;THiDH@gI#F}@zH?a(#Trhr z*56&XCUWgPDa?k|P^^%^<;d0Q{4g6qzqo$gf2H1UYe5{4aWUpNttKG8$xt{>@)A z@wFg}NV6KGhBP}&L4Qay7-SA2U=roHXM=N=S1* z$avCx0WyFzyH3R_Ak9f2Wu&&jNqQvVkvxwC z9%yc`YRC%PrBjp|`^+<^lv)=Qr#Jtn~a|Fm(BA0+v5?KgR zMPwbwG$MOmg}si*i6AvZZUm_%@;XTQpYi>$&DCg!swD%`fZ4eakI%w0!FZ5)q`42| zaUxA1?-AKy8dffmlRzr84o!aU0%?0!eBby2WHM>?nvRuAnsFf0NHZU#3u&4`W|F4x z8mwH>TmVu_nrA^$r1=MAK56<~i6sv~j~NEMN5L8cK|0a8t5%j-b?idWQeAOk2rH-fCDih2uV6lwCV zZy%gRWEjZhL~1~4hWEM9u+OMdUG% z)kM~TG!f~3GqOSCbdU{1=76-_7_W^lKyrvAX5x-dWCTbzBC|kx5cvosNu={FAVE93 zV}-j`8OSo4!J9$WQXRet(nOl}x3&-dM`R$#es{kg zfin_mZU9+Anh!zxlV*=Qk$KXL2N^<|g&>Wj`4^;|H2r5`c1UwG$avCx0;O8Td{-oLap7y~k)kuEwyjiZ(Y>1{-iD~(4w9`Z*eF9`HkzYWX ziKOmD)e$)lBxn`iqn-lEBeDUcv~^t5`#!848rQ`jLFf1h?HQ2qr1=x%JR*JOv=43| zayiI6A`OOU#sd88a)0|^0cp-Ogfxpl>PfTtToBR>23bLxdki7X4!AyNprwM?SswkiLb#cL3Sgu93(@e%fpDJc{XD|3#2b;UI1w( zP2POeDrrsy*+`lvL57oNvqx|SI4FJ}911d)H1j|@sYd3s88`OHwAYpBe2FIMk%UL` zJQ64|WB&uun`kad9z}kLTn)08$V!lABHKNN9e~Jikf3dRmhS_}A@VIqXCnD^csfU< z5+q5a9;9OPc+2e{#|eb8dMrq1s-K%dW{~D1kXwlC^F;gLPa+dQO74x<*h?S@oiCHa z?ViND0Md*D=|P$$AZ4VOZ&1{gR zr1=?S4rva57JCb6?gXhL&5t0fNz->RPI{!N0a;6$??IN4rq2@G@kw(lNF!;!1qtqp z*Uur(Va2IN)^d}#-kZ1ePCFiH@JPK!>O4~Gk!p`rd8EQ4ZNQp<19!Yp4&m)0H znzxbsm^FAavRXxZ!kZv7h_rhi^-QD>$SfikgUlha5M(})A3+un>HY%hhsbFl%ZSVd zSw-Xnkkv%CsYh)P83?kL$TW~;=V(g zQ6Qa2^Ds!;gX4GMpFk3%$-IKoDQPYN=|P%TK)R47_f_oSq&W_x8)<4l`jX}|kQ8b5 zdJTIxX~u(;kmd=H;iUNkWB_RnS%y8FG*^L?k!A(RB+_(x9rqp5oCH!un)^U1Nb@bo z4AK-V$9;!17lPD~=0%Wcq{(>$D~>d!ATvpG8%P~#J_D&G&E6}p21zpkWIkyYf-ED= z-yn-f)Avp6;iQ=XvXnIMf;5q4=asnckmgj7b)rvh}-~@B=P}B#a8i_cYhbrsd`TbNzy+55J=wr@!j|*kR6E}+}J+2kjRxF zONe|3vR2oHnTsCpVV#nu3S>QL-UVqS%|7qrK6prcpT7vCi8L!gI*?}f53o*2a~?=% z(kuhnK$KSahza|TFn(mV&!g*01#gr_~E83odfG>brn zkS6zI++|2p4pL5<1t2A)*u#3_Yp>?)Anw6nbdGLtmh zu4y04CNdJ_Zz2zZlspjM0saA*r`2ob;<&G{nn?2i$O6**1yV+u(yx(2(#!!FMVdcA zR*+`kH&{)ixff&=X?_ElOq!$DBG06`8)O=3)`K*YrvLwtG1ANi*+`n7Kx#>I__w&D zmBe@KJ3!`>=0}iDq&e(6)EH@Q2T73T2au(t>AMc+dD7et(m$!H$VQL~A_p|%PTeWKOI!pJ%!{w( zg&_TDm-rE+GiiGK)IQjq$V8BQBJ)A|68Q~etX6^<*C9V+T%?%}GKnyddPM}Z`^i|6?&kg+t1=Rk%~ zo__(UB2AB9+XvH$oC$I}kvSmqiF^w(L-S+C)$2FxB&0bHq=q!lf-EIX@H^hIsz!2H z?d7>#b10fpkCZ4e{pp96la!z9K}v{x0MegGVgu%a$T*NPB2R!+5ZMS)w|#sr`u~Cb zkVbj~NIA{Ldmw8`v+JMjgP(|;1(N$XvRJgBuSd5L6(rF!$#bx zNpmX5GSWN^(w{UP|HeB7(wqu1gfxplnp7k6RprfTQs+xF36JD?B=ATxzC#_J{Q~@S z{0H}Unu{SI4Mgq)Sw-XvkR~GC|HbSOsQ?Lfh|lsOkdn@E(mKcu$|)O1fQ+YkzZRsD zH1B|1Mr5~Exxt-8P6t^)o8<;$ ziChg*LF8?aN+LUNo*PsXIUS^7$N0QI3DQj2_zR?x=Kauixk1iD@wGeyWNRX;LG~lE zcTR55m&g>5ZklH^_IE)_Nwe=3xj~9FRUpGj^8rW+X?nIten>MFWB_SC0+~db{M_82 zOf@p6<=(tk>wJl($|DsXDfdXJ5;OLz5xt4#;$x6zBE33b6%e@sB-klFgCBt85ZSL| zZjc~S1ya5;K#3W9pKUQxnv07-s)#%ZGL6VOkZK}(cf#7~5}(skLGoy%w}VvEoW2FpjWjvi zzar86Xuzo&>2RvJs?;$l;x_+w2n0^GzU?G>Q*FW>KDZ z+Yy;3%~+7zi98MRAdz6F++Ycjp&&~&KW1DHgDfLWtDUi{lV&JLBWdP?tX7TWFzMyl z{61#RYMn=Fm6-mt>Vi6?{0s&uCvrc?SR%iJR1i68m)sz^Ydp_)gEY}7egv6Ec|Leo z>;R;x25B`vzTVe@Y)#~V-ExDjL~aLJtNAhG`W+-mnxTo@U_EITfb=HKw!7yB!J%>{ zkvu27JXdQDMN{RG3MHmLXQSm3%Fml1{fTtn19L&-Dv$;uUxDf#Nnhi6GD?nzDW*tap)krq-yo{A=HbhgZ#I!tQZ&VA7 zbSlVNA`3v8iL3$HK%~N0+E9(;8Jq5ZzlWxo z7fsS536JC{F{2oWSb3DyYLL!E-UUez*@-DDDDTL38mbNDtEN-Xk|C zB60>u36V!Y1`zohWQgX+jAB4foFPdw3#42%lAq?*|IK00@xME5a|a=&?m!3vt&Y%^e_9N%JGfTGI4O z;oeP}J3!WxrWs@wY5El*V~@t`@J^6Q?<7 z0@KkS)zH7|A8%b~&}{mF`F&QA>q+Ob8u}dqXL+%H)8F~9+PSQTVojh}U-rEC`^dHV zL17QAhGI>mSmt+UJ*&SvtCw?G4aG9=&)c+;ahczp6}d(@m(|$3%mA78XnZX{3(`z= z_zg%6X?8dm?~aKK1bKkS6p%V1&w{M+YNMpZIX!#J_?#A&T(Ia|{XHJPp&j#MH8l2f ztV_n;_p#9%BiEoqLYLK0p3n7SovpDRb}p--Sd+Y1k*ifn7|UuX)_D}`s}HQaDRSN8TvlWA@*T*MN8@wasSnObG^f2mmXc-+NY0M&e|vHxNU&FY z*IH^cM1BX!BeL(IILQzh0n$WdCddkk{t8GZivBCe8q#z-EI0U`$ViawAB&H4CP;50 zjUdfhbtXaE_QlzcG=o7llIA9mA*6W+BGfkKoV4A-+(M2&F)9$2G0>0 z39^jHogj@wz5prJJeyH;I|?;Vnu#F8RU>(>^X8&Xvmu&VC8p(tXs4P+`YlKek=>5Q zdj}#zKxZxljiQEQKOk@Sfa3Wg{#B&nO zh8fqfhLGk)kV&L@#}LwVItKGjn&S;2&21nvNVCci((H6Ba!8t!3?a>IkU6CJ)DY4n zN|9&Mj5LHa_kk=?jn1-H3Dr99qN(yog-6Ogf^h#?l_efYdL*I5%+4A#QbpC$We`pd zM23J=6PW>0OXL-h@^0~}3ywoJC_lv@8LGN7K^jSOAIK*}J_p(S@pxtKI2hGSWCX}E z&Ab`cy&xIVd;`)*n*D}g7bQ&vNE2xmfRvKv50G`FIp}z-0@7R#vVkhg5nMH6_$d|l)}`@BaQJW}tGI*-&U zF|#ugeXgKtnFUfw~)WKPHp9wBle z$ooVVfNUhP5u}@D-i+(86LFS!B3`T4f}}|E4oD}`>~s>I+mL1?NP;vEf(#%{Ge{58 zWQOA%B55uK$<XkWr-B@?_+PG($luNHZIxlr&#~OjeClRFzkC)jIF0@koV7$~{u* zkrIz2J(5sjW~avp>`_!LCxVP3G7Y4H$kQOrMAm`~r)+dO1^J;GI|O7BY0d(fLgXHh zdx*RRvXn@BZ2QwR&t?>TK^jPNDacIHJOi?tH0wcXNwfb*JQX3$IFR|IxesJLX+8y6 zM4Ha0VjFofUh{)NmXhW=kPf6-2GXD!sf`M+e)4n%MH6_Wc?)mX@<@Y6>OE4YB>oiQ zf-U3kSj|(28hr}U`ZSCkKUPEU>?*Ab2Wfo@5xKfLm(|d_$P2w#8}uneKj*R#WGJJmgrN6Y0hOe6l*fYGEX5Q*JI9QH8wBrfpn&M{|jU|t(3$l z?4qRU50X#he2_GeyFf~aybH40tEk!*=X45<-Tao+>H2QH-RN*mt%k;a361@;ssH{J zxsG-&tFdFh9%KOZ@M(~-H1^LxhLEQH>DV_&lL86$jo;%&f#eXG36e)-DM)7`>p>Dk z_8J4SU%cfqkm1yyt3ax#KTm>8BF&c|Q;6(1HaED5$dMrPh+GA-R_Dv4>?M!|r1=+Q zJ!uXchv%83sRRl7#m`X-Kvt0ECy)-Rk)5R6+ij|~T0~RjkqVELD>0+!c?O;mQ&z`- zWQg1YQbJ@kNIj7q&%_f~8pR-x88jEuK$=PO638D!T93y$b3uH6?hTShZP;>&Axhv?yEx)%ludkt-&f1*xIhm;{m`%`A{UM3#dbLF8wUQX>0IL_K@+p42%t9Xbh`;pDm$ zWDfP`UXW3wSq4%^n(sjxXcgpDU=SPmtL}Nv^BA|HVS z3*&oKyGhuwiSz*}*KC+kTm;gYGz&n+t46Ys^fK0r50(D&K39op`Fj(KM!M5^xOWpd z0;Gb-c_7V1?g!~gJ!}AJr2PB|(wj8ysn*gswgw}9jjc@89x$k!m%M7FNPS%P|)25F-F35}p&RhE5k((dojRnj%+!=dv0)$z1Kl+Nh_dQO;#G z6l)sAGN-1PqKr%!=11TZW?lM#_k$jMO6l*9*H;Q$UagpX8kk^T<1ZgDlD@YTO?w8|J zV9k%o#&D2zq^SmJR*kHSCU5POXjVm&RAO3w8SOOCNY{g`BC^*N=pm6|Am#b-z2#bv z4b;PDK~j{TwIF#<$M?(~r{)Gb5-A1gO5`$-B#{?DHtN{TC^mxhCd~m?Vt+nd-ucSd z8@&7^G#jGHQ({`a0PR%JNS^?yB=S8-6_M^&VYexW=VuH^f9hc^ND1X<6-YU0w!IqP z%Ox@dWD=1ZK&ps*0MbdvZbq@|G^|0=oDPyujf}nC%TJ)$5KS|_ryrgV<9qM5v`3k} z_b%OgA3-ehV>Pr#&7eKX?7fj|rE^&g?T6QSvHsD$_Xp>)8j5v2#WH(uDgi zRzaP&QW`XeqN!IBUnw_IMVY_Wbf*4Ao}+FE>(FYbqHf}f+9vT~?>Q6qQ_5-? z$SNXNfHV<#45Yj;e!utxWHM!A^IPy~7tO`~ATvpG0?2G4*MQ6;vJhkuk?%pqYM#w7 zy55TSR-_pYGD$U(=W1^*sx|YXsZwHEz8URQ&`6hoR1*0cq>4zvZFql5WGqN&DxR@g zkOu1WyC5?uW1H3BxjJbM0Qrf?c_6KxiSHW^fpj9W4x~!6VaC<-cI+FZnFun2G<6_7 zNb@sDjcPPwUREnKhoUL>NU0LjpTav(QIwx^K*kbz9HfFsGe{+oUUy=oL?#+i6wmzQ zAQ>9h4`6oN)^0?2%tiy0s*Nb?fNdqn;LX(E!k2k$tDTmaIj`7xt-24o{?Hi9&%M)Fhc zq>TDA4rJ+0@pt`qfK-rXCCG(Da^~g+mlEj>GL6U-khVws_Z?Y@H`00< z*K=?ssn36bWQb%RK<0^5g47dv24p7n=U0%1o#G=+&cjL}%>5caS_H1@kddA{T*l)3KXTyl4n%ws-{VOEog~ zgqNRc&4y^Il$e%F(M|=8vNF7KOktUF7MDiZR{!F9e$UFt_2xRnw226s*$lbLp!;h`M#NuK(isF zd9(O=BX1G2S z@&m|x9lOa-w`V~}a~jAZ)yUYJy!%gfMC9*y)Dkj_MQ zeG%g#awJGMB9lOpMD7OZO=LMpDUshm8V-)(NLDf+Q)==Ydp^rWWKvB5#A# z5ZMT_h)C*X+zT{^CL3phEG5l-Ao!zY!A0%DZ^_~>f`K%UW}BthNl0@f$ZFDD2GW@{ zi$K)be2U^;gNEWlzODZBS|GDW4EF|O*Dfm zK-Lmz{VH|DS89ZRU@@gi@x)|kJOIRPY($mJlNi98IFAkqlZjmY01Ng{i%#QBHFu^=TxDna@axd)__$a0V} zBENu?6Y266PEADmfm9Hg08&ZhHjpYJ^&r!T{12p>$Tkhwv56cAQcGkM$UGv~fz%Os z3S<$H&p_&luI z)3Nc9jsclRJJNL^{b_%G0;HZaAA!71B)$suwPNFI?okj_ND z21yX<@+s<&$Z;S^BGW)JM4ksJA@VCoe-Feka^Nf z0jVO*3m{!c)A~#7>ZIul(v37%gVd1bb&wQkI;_F2PMTvtN=S1fNF8b31sOn^?Z3jV zPMR{1GSb`$vP?Cy3aXGxK4aHvg^Q-zBUK)$@JP8wN-Y#`0+ zAUQ9^*H_2yu;Px2-%m?HI+123NFHfE07;N$=XF?dq!|IygEaSobW@GgY6Wt@^&Dum zh^D!|9nuv2LeW=KGLKHP%VaWi=G*4vJ;I z4;i^GaW1Q|d8q}-ycDmxmq5nT`uY;2KWWgv zbbvI+fXpUxC&+z7z5tm|r1)pto4jmPw2o(EIy8&O^%_Vu^{4GGcsD_sfgrO~6JIHJ zQbn2XLq296@YtVQ;4(i}LlrfPD(cI@V~&hmZ#$ROQ1#BXt~|7eAqC$*HTs(2`pw}E z>%&-9L$U6nSf4$bTNAnZJD1f^th=qtjHDI*H}Haj+auQ%&Sf>^sqY0X8gkuZT{+IRXW6jD`|GOw*SV~Q#&s`^YyIc*OCr~ZU&Cxz4aK^TVwvCOx6kG> zUrU_JYADtmiZ!(V<#$A`znsfzDAxTH%hbkaZ6#Lrx3Gs+L$T&ktofP7XCqgYb6E|= zdcccyynZ|MRp+uAiZzd7?U%oJa^%|L_ppanL$MxImzlC*bCav@)>ub7m(@_Lhty^2 zqifx}<098J&SftPy~`F##UHHXWb%W5dre2TUH?z%ORYyS=5xU7a^JwmZe9p0?5 zZgwuKp;(VntSN)vyfSkA>0DN0>%069oN{Szxf5ib?kz_29>^-vZ1ZPs@Hvs=LDmtu z31kD2)gS|OKQt}x^%u@RFUNQFi6CXFkrPOHYuPh}lqxapEP^YK+SvfIKt~bZAs?fx zn%@gLQFqAn#&9mIhO$~mSv9{GG;*zWE~_!!39A0ayHV=TBOvwKAEWsYBte>-fABMC+K@z0-38b7f-SInVdyu9aWGrc}2g#@=KJSmyyqh!Ry?TaR zY=7Hr5LgY(`x7+p<_sCRK6fswp_Tij7pt}2K{jt~epe5Ek&)3*tOXRyoFOAu59hKP zinWkpnKR_88f%bqSq;T{iei~FWaOIcTvlWIIv1pZ^4tK@SMzK{e}GhxX0J9Kg6Twt zf!sjk3XmEiOF%|>J3tAo_w~@sA=mD0QFYXxfgtlqGZ|z!X&wTpqVwTbAk{>6->gH> zK;#sV1=Mm4$XIH*0c0s@ayRb~yh&sL$h$_U$?ZP1M7oAZv)+ z3ersE1CWZL@ps}o<#Y(vQuH#ANi?ooK!T<59s50yEr{&AMTejpk&{6B6PX7xS;uAS zXFbRe()4NHA(%#*>p;p$^AX5Q((IkvAsA1Zb3tlJ^8!dEX*zU3wUDL^WGZRy2U(;V zxf7M6@B9urp|c~JJdcRykYTq z|K4b5q}{hf%QWw!K<1I=UXaI$d=1h-r0Z6wIwBK6mTC^oD4qe?NSe0zMZpcEIR+%> zm3Y<1kK>5AYF;v7l#Tu% zGpIH$11TlVb08-W`4?mgkps8Gsw6TMq=V+!jC2Lae9~;UeTSekX-)=NM4GuET}kr; z$Wqdzcfd$Vb16szX_kTXCQV*vR2^wf0O?PfdqCEb=39^k@JOCV0wwV?($jR7FlVGWdPXYPF+8_e4V@(x(OJTrks?=_b6E|YTb}V^wbgUW z6z8%UiuEkTGH0a7b)R!t4aHhavCJ81nZ{b?TvkJ|mQXBnMv7d&IG5F!QVn+52|EeR z`w<{Dv{EWS+P)g!$7X?aB(f4@TOxmgbRkl>Gwy?4MKz-r;T^>E=S*n2k?UrVInMD7I{ zMWhj=g2=YJ;tZhqG1(XdGMO~hAeE|-6_79b}q;?>fwhV%P2p&3GA7qDFJzq$i*NFh}4196Zsirs*c@^qHy;P!3xrx z2U4vX$xi}W&QCoE=ci5yE`q1fP8E&xSCDB$QhOj4kx3wVC&u&h6v!&-;V&SKl%HOE zBG06m0P+WsCqdf27GHxugX9r8xNC=CmX6(wVme3{(!2*UPc@RCJZL#TwIH0IY9;Z# z_c_|5%-%a!_uj-_;ofUCv`0Npdz9IGBUc~ivKrbCU+`jWuKVFg=dv1#RZp?Z-W$0t zcP^`;ST9m6v-d97Sob=Y)ljULD3;lKBiAzLvKr(P&z?cL(d=xqH&!LpMlnc+G$(`f zA#x4K5kwvbDJAk1$O3QPtI&&R@9n$~-f5ES0FZUmpOZjFk>*m64WzjjBtfg-6_6y6 zwICTHx!tg16WJf6KaoKor9>uylo7cNq@2iNki_u#s{9;eJdL7tcdV~n;!lM30GUdf zejwKonE>)QksCo8iM$9>ubDUV-VD-2n#8`?$4GN5$U4$o3bKkckAiF<&8Hx1NYimY zjB8nZ-#7@QnKa`;@<=laWFu+b0O>-Szd&*Z{P+EZ86WQ?6(GEmlzXJqBPAY5dL-eI zJSFix>SbEXW{;Yud(^%?!ad4rXe}?LwQTmN$koretcLcDSG-s`x^JA~TvkJ|UZq%O zkBVFuIhWN?tk)=(*`royteMVbH56+Z#WH(T)1QipMRXoYG?*m zSXYh-gr7cbm)GFm(uV&hxH1{WvKor@CdImI-w}^RuE(9rYJ&aGpE!QfspD;nE3FG} zpzP0-#*cOiU2+C6*lL1!p5L;ra=d2pyfF9T?$>D#SG!nNgI9x~cYc`Xh|C5tc@9ZD z&keTaPI%3<{LYDcTo<(*6og~98XEiCH1?z0{&7v@8sS`4LwSD3y0&n!7ALxABiCZ* zvKn%|Yh9beWwLh0ZX14%Tss$rJ+vBfHEJxFg72PL{9EL@%(<+FT<__)jO)W6zwR5k zK6fswp=`XbE>m?2NB?m_ESZm@G2uLP^19IG5E>thHXOw8pA&E~}wf|MOx+ zt~Z^_YADvXUaWx{t5r4}m(@_L@4Q%%>j3An8j7{fi*>ZdI>))JhGKp1#fn@Doy%${ z)(;eG+iPo^x6oL>JD1f^tRE@Xlnb+8N3K&32(w`|6sy^b)lOrra4xH%SU-8OB3H)) z!&p{h_vV3)nE%r{*#FdX&$Sr;`I(}JRCe^UoXcq7ik+H%v94~A%PAwg`x^g4&Sf-V zOV4riSUc;r9pn7xuMInhCkR$Mm(g(aU#)9vW5Lh=zHay^a&6x`9J|r5>o@D#FEj;{ zreBQD_GP3c&Sf+qo8pL(b34D=b`Ey!R4>@3t$6S+*`3R1*tNmBwsEdomjCrn*KLgni!A5fg;l`O~&j0gemzBV#|zxr~Nge_2(#3`YtL!HXe@SMNWz<(8{=1YG9FXn0)zT9-NfOE03UbDYa)*cEJr?>V$O(6$)_ z9ro>cb>y1vTt>sLR@Rm4T(iGD;k?MT)VYiXu2{xeTUW~&TkBj#6ZXIRA>l|(5RTr) zw$pNcj&&}hVOLx0%D6sHs@eQai4+8sL4%TJ9RcwAfDVcOc)WgBfWi;&SNUmOu zrMBY3HfA`N(XeYv>uPyiuQ->{uxl&p+BbePb3zB~E;57V&Sf<0+QzyDIoGf^%hpD&8s{<^c5O?pwihq_Byzp$ zTt>sLPS(|OR<}Gf%!bjhYr9Rj_HizwVb}K7wWAx?^1Wx>8TByZTt>sL9X1hbymJ{1 zyE<>eb)9n=4ZC*SgzG-%G8%U6v&|5~?CP=!*T>FfH0;`C6RuyK z%V^lO>n2=zhlRCaH0;`K6Rz&gWi;$cSl6zuHo9Fl<#nS7e-3aiqhZ(X)`bWD_UG(V z+I<|k&UG%MVb>nkwVQKwdGGXPk?SGnG8%U6Nv>z|FWM_|t#K}+VOLk{YI%L_+BeLh z(XeZ;O}O%%%V^lO_a)o&xVAVv%&pO|tI)byeh;#ja~Tc0Qr302o8=d`xhb}C)6Qiy z;T)a=QU$GFV@0-|mTT-`*N)L}^kVChvtP{~$2=>M@a-(;G8%TJt*Zq6$ie@@AE~i# zoXcp~m9Z`yr|eJrm0w&K#R`rHvuZT#%34>#xsF+R?QfB5cjq!1b{$|{vSXk0&Rwy0 zB_o{6XxMe2b#-^K61$YgTz5E^(Xgwxbx9A8{Jh7KsE4bZ%V^kjkabB9hyR_8y_?vs zf0zxUVb{SFYyaCHjh)#BIhWC}>k#W|xiW8XE~5$O@DUL6Z;JgIE3xggTx0*Zc8rFj z_pz?+Tvk7R;O$s`_B%2hsnM|OQ0r>vTsO`?qL=h92nIWs(Xi_<>uPzV&}T{}j@(fe7KjCAgg=RYWsuu_f+M`|?eI^4Qi?sL9#8BN&dVIcJ_ z-er!k?X=wIMXnvA;pqLX3m;S3pK*I$d8I_c_b!~vXxMe6b>Tnu=ex{`VUcUsqr;IJ z4ZDuAu9iny;#@`(j`Vbp`W9#BXxmQ9BYn-aV>BFnfOW}8JDs^lwM4?-#Bna8Vb?(G zYPrun2ZSRvny}A9K+0QOJIB~|TJH1Ht{tP{=*L=@>=MIHnbB7wVRoF$XxLS1U7NYt zd3Di8u`}du1H(QW4Z8+eSIZ+Ub}pj{M|v_yIWGORbDV9b<&oCAc8rFj54J8D>ApMc zhqstvq#*dsxr~NgL#(UiK6g7N9I4TSEe!)P-?jGpe7tR^hCpou$5_=-l+qsN}U1iqQ@<@j`m(hgruK+RsD#0J=3AUY< zNBWy<$7neEiPj|}z2=5<@#s!Qnko%PYBcOR$+}wZ^H}FHny{rCKpIf9exHZic3ST9 z&#oP#;pit@msHfl7rc6%M8fZS9TfK2XxKHvx@7HKf64LvBG-w|Wi;$M#kyJ^>DkU@ zG~q~VKs0G%xzDFLm(hfM zz6r$q79)RlPP6T_+~>^(htZ9OqmQyKsi+6HnTnYYBjN99I+xL~YqWLA+8O`$u-MbN z3!KYn*mb&fwLH=roXcp!_|Jnhw77?lvF)@x()=OeNR5W0kF_otY2%4|;(#C{o#)1G)$%*C`<%;Y*mcGxTu(cf(Xi{xO}O52E~8=Bc)5S;d;rrjD}t3Zo;+7xr~NgldMagES<4*`C6k0f4*}rqhZ&1)+KvOr?VH_7`gTu z8qSx|u>+(&wK5#Cpp*p<6y3C!<{8aZ@ z9Xp|Y?_5U1sLE6FwI_)}&YMflV4gfJUM!>+5ywYYjr?4GiV za~Tc0t|r%?Q+|ql#?jNcjD}s)$n|Tdk#nLR9^_m`!>;M%Dr{U7+ieCqm(j568ggAa z@(SE2BpW9?m(hgmEJsYXS_h|`K7P`9=Z+rbl4n{z?6@Ia%MQc3!CybHEmz^n+`swf zQrC{v*q5FIF|`(ve?W{gBIy(HZxxB00g@o{Ac)zSBG+1wB#~Yx;akB(&Ic(W@;pd? zBAX8nzb_iKJQM^AFcxbrNI8)oK*ka&J{jjaB3FS_D%pS3=u^)d({(UFFvI3>IId() zPT0zvl;fJ~tfp^FbCRdI-fD)#G{Zg34OU|^C9%4Bnrcsz=V@;A`m?X6xyjR<;%RQS zns}efJ{;KVvX@McY3kLd74>PV`^RIw9?bewi+yZ)tupJ?(*8{vDZGd<0HR?{P=$Kd@d$+1zVqfu~7&nuT6}{`F!#Wi|0U@8Y%dwAW61 zPqWBs;&YMjX`b<7?doZs^)v;ZX0g@8YqikREU}vS`YQ4?&v~)Zp5}S0iI3|LPxFG+ z#P0uj>Ja+#&-qXBi zHSzqM<7wWvn)v>+rKkD8YU1NM*VC-Bns}ef;uQ$j$rR1tGy3$q(Gy1Bwk}43E z(pHKzT2WeFEmCihdW*FO#rdMc=v?oMHEyxCSFEiRYbzOzk=6FH+Fn+BmDTpL+Fn*# zDNc&FIH^oCR9vV`$6Ku9Elx?x#VP4aaY}kxoRPj1XJl-}8Fgsyi!&NS$5E`kFV3o0 zdtaQ@7&?x0QsyI_l#!>k4$?{KUpgt?biVX2ov+?}iI&cnT&45XTOd767pkLBqZP{7 z(uL~K%1IX`gG0_e<+#zOj~{*7q;V4`9690ii6zHO95wosfukplEFC#%>?wmMoilF2 zm{X3eFt3jsUs94R?RV(V|5!@;9qI}tU8JFlwA)&OX)V087G657Lr$wBt=&#*x6?Y% zjFd-Olb%URD;Z6CCMoS@l9ITLCOxC2mC>YUH0c>ldPb9;(WGZItr<;bM$0mj(!OX} zW>VUhl=el-GLw?AWs20RnapS=vpS+I%Bbb38#4ObN&QEUuNZyK|20=x%}Q3YlGReo zYJRf1^s<_%tY#{kFEgIiOl37w*?bvUwm?RfEs#RZ7RWqj3nVL9E$D2a%yU*tJF8n^ zR_7qAbCA_J$m-UR)j7!O){xb$A**wc)y!v$w0}j~Kds@c)^Jw0%q+IdmS@9TNXLyn z`@C`Ij2`uWOinr}sm7J-4z66PaplShSFVe=aw)`>TQs0-I#;!5UF@3>$x|HX3e1=I1$4FOEXt9RkF-i2``{nPs~nDkHY z%wW<#y))x}CAmuLdQ9tjOzV2oyEPorzqGEsv}}jzv}}jClJTZ>J*H)g#H-prU5{zm zBGc(Y$-SNfz_dNtHo+u+Y1uZxG*>C9jkK=Yw65ENW0S2EOxx47oR;l1 zoz_*YhXpVlo19X?H21o`)4CSYx)##97Sg&F(sFW5qh#G~f@0Gv?Vqj%J%*%_XwjuL ze{wXztC~Mu3u#>oXd^%gExFgD3z$0eKB)&5 zyef5X4lLn4HmhqUt7|5!cap5GnXKMFvbtundjH7k`pD|~$m(4qtLr1H>m#dcAuDSk zla$`$%2hG0+^WTu>pf2MuJ^cd6^ko3a$LEY!IgTYrzmGP)3{QH-08s7D?LToYc%OI znD$RrEtvLCRxK8aMw1l^re0a0VB*b}b%~`VW6PJj6`1rdUsfX)o~r{~Nwj=fd0?($ zaiv~a0a&UswtT63Fv&~4)Hs-AGGA5{7Pm%|`wW;ylN!f6Dea%sIGB!2Y8;z@_C@aJ zU>a95R3Lj8HVbhS$eTVeX}LhNRv_=WlSvsbu3YbNVT=(h6SH9>~ssEz#u}tw`1iB;#Nj zEox74pVX5!t~7tKmbLeCpGGttx!kd_70XN)$UQ!plsh(F)x7BXEs%8%oyL_IV6svQ zWETLF3M-KP08GalwJfKCWK#AkTxq??ZUH7$Tp-cFv@g1s6i5ubsuiKTMS;Y~6i9o? zr0yUE(jJ&blYJ6Qqs7|O`IY?zI$6JkTK9!o?S)$Hg>sI?%}?@DC~xL*8K;)j zr&qix*)5d2GniCPp{$-vp{yQU$!<|7YX(evsw+iz2E2*YS7k2&)1Jyb6ig~icM!bo zb-l-xtlxZjHwh*)ldsj8FExob%IekX%$NGh6v~=`PIDlw;CWBv)oetqXw?A zWu^k~JzVL?b%#r7 zMWnRCQo3WNWDm=vq?&N0xze30C1<@%O838%?s_TR>r#4xO34lfhwK6=JxitJRDfHp zj!jNg$wIBuXirS(Nh>8ME$Fl&w8jhNRDh?ivfCEw`Km}-&J@XRi&ms23#G>Ks?1E0 z#7GuO-Q!i6{b-LZk{Hm*EEGx0c>XK(tUEAtGV-FRW!b%P@Nm@$uWJuXdnG$MnB<^H z_I5Ct!6JzPCbLi^ErV$fbmuRU_Hf{l-KR*_G??rbg}Md{b>$W6dMlJwmMqjAy--(F zk*?4pSx0ypuKm+BQzZLyrbyRJk*>EQT{A_p14E~+$Zne{(lt{gJ1dxGP4-hTxq}tS z-kB-VHB+Q}agnZ>B7F~0q-&u_-}e;hS}2ldTA3oLNnAQ!Mj_E7eIIKo>Dt0nsj+(z#6O9b5O#WVCmtbbfWmOr~_- z&}Wf2yvhnlN!Gxmp7mKIn2uMT#DHl}Wgh~Q+0rMKV6x(Mr@;YP8-TgVA16Br}-QPlj>jdXFpVU$h?-$;l0eby-_QGFxEMgCbe0 zILga?t4LR9T3*fQvqN0T`c3PdQ=b#!Rmq<|F~l2nWwL%V`os`dI$ph(>9a$;wU_-W zEvp+$=2xF5g30`*WyOQZ{F=`$?Gp?zS;_j0FUG^&YCv7?VYMN5li@uzwjmM3$aPO@e8Lq}suBR&|fir;T`3YeV)8JQ~pq z=?W;)YS(9x8GRNBhpgWst@dbV(C2y?eXfT`Es|Y*u9qy9=LfiwoazUr@VZ{%Rmqh; z^#hYWXQX8?sV05?2c~hO7;^8yn?;w?j6M?tlPcC{f?!ftcw^?~BO@*2Rmo|w*0??~ z%;*zC=p<{!TJ4#rWmz+L1f@NQTGlh6elr0r>)2u~E7Q|vvEJWs9CkHStg8n{Mr9gT zS5&do0p1+DmXo?8;Y!<+-3&~#p}SdHPC*pk3Zf0;LdDhQK@b+BeN;QE=4e8U`j6SW!mFyAvWEM& z`k`_rtFy1qXu;Ggxd&6P&Y(W6&FIrwT*)lx=NfppAS+6rrA=4xGtKdw5lLBz&afR!>%1#;Y1v-#O^BVZ18!tWSx-vigWd9MMT#+5S& zzJuf9;!30GX+*!kgKyI4*rZBB* z!yy%suk)rmA-pmo{d!GO_e5ODh_W&>V3I%G9l>OM=spQ1^QJpyQg=*T$!zJ~iEj=` z#`Q}&&}lT859qXIotfzApzgL=eL9FM$$h@G2c~(}xs09@M!TMV=9SdXypp3qnJl&^aPuACQF zQX2)b3!s=>uJD1A>ZI=R(Ua>Ru2d)Qj#0WUF0R}tQN*f~UV%vt3bgJEWE?1VSrrB9 z)z86Duo6Q*2LqF6`EmvUb7Mn$l3jf&1STUdly_cO0Fq~YW)7x0-B0yNITnhWeOyU3 z>2n$|_38@MCox!B>d>9FP7TQyZhV3e18dujsVbq?i3OKZ7 zSrz&D^6m*&Zhyg*wkPk8z|<>yF}6w916-+B?$cm8BDqh4Nrma>jo4zP+VxpDm{ezh zR3biTl^FU|986*qNHyWZSLwYzod%P2sn4JD^$9bsvF=+YeMK3OOk z$5Gz(6j$oi$}W@>GH!V~a(Nm6rhV4RF4UbcCA%m-E|VUlkDqMk^_DA1t#^U^#Vtv(^qAG z#s_;ES58Y{8dvHMOycUhR500D)3Wn}$$0fiIhe-PTE0TAb_()S`MxUyIN#*Dha4?ObwWiOh^YuA3 zt~8Uni|SJ*ysE9}E?T5__9ERy^~rX=KF`kAf9HXlkc^{P^Q=#qa66Lu)h9||l4pHV z1SWabrzE&B$=Wa0eX>}yQLOu9v1X%KSF(P~7B@A?fqu&tOsYh`QH$fB_Eg@H;v-y{ z4}JauroGY(>GK!7D&vsnFE}B&b!o1`T$!t|2j(iA1#=a8apiVBa~1a9T&dU9nqlg2 zwFc&D&0K|*W3JTe&K6+qUV$rj?>1K&%{?_RESxQKr9E|f9hlqU%$0iG4riE-($yN6 z%c;2vGik2GoA1^+&XDeXt+|p>%DW;kS6AjrMv2?0eOGCi^fX^{ikpb_&6VUu z-d7kVvnr=0Fn8ZFR}xJ=6E-Yz$SCDJgjY4Yavm~FWaPXFdgYN*idiH2zp z+@5HdB8!{N&9%D4)9U|!W2a^eQl7;;|)(-?9J2h$jGuPxB$KDg2t za>oG+vy0tX-hJXqdSL!Bm;EdYOrq&O!~&CO`VX=2CR#?M{}2mI+mjOyn6@l;STKzt zPyN7Lh2`t>CcHfl?^{4o%W@~it1?P`1`npAl)E#S%$9y*8cb$OpGM)0zqY4yuFsqB zs;fkFMMo|>D_H0?9n~D@OzX3y0{sVIxY8Wxtmf&u2@;)^TpRDdftB3!xN&HTTXPUNItUMCUf=g1AqWOHn~Y|`JJ^U zl{fK#BuIiJ2!gH;!T90ij9-?Uwh_Kb`Oph=R>Lpjr~MuZ&Pt#*1$;Apxkm6%78<%U{O9@X+epqOm(&=#j5qy*&g3?w)ZDm_Cc$1`# z0DRLj>na1&mhRf{OA)%Zpj1Oxf7Vq7O@rE-zEjYkY|9eydlW3{0?BM&-U47{KiS)Z1>*?zyc{*3FNFU}rT>l?bP zi<_YH*E83t%*_r!EwH)S;hPQ#bF;(zt_^NZGWcd#RZRG1SZgYCu6t(gE%;?vYcz9e zL8>;hZT6qp79YP9YukBz(|paz4J#3CAls(nn+ngi-}q)&Yh2rEV-=&_(6-w6rk%x{ zU$Dlp+)R~yuZswzS{}M!z*SiC`3~PPJHv7-a zeFVP@Ys1~P{YbU?s;LLoV3n0QwBei9*WB3fO|d^%ANs+jz8`GrGj}(vQnk!xP{22B zUo$9R4Xb7T!IZrj6p(89nqoJH6Zl*k)|B`U*00Q^1ay|K^{XGOUs=EktfH;V)~_sJ z1?=D^E1PJ|kbzXgTE8+w23F;UHMM1i45S*?`juT5#yZ}x)~|lBe)WU(E4w5Ngw@yj zm0bvix4E^i^((s&j8wy_u>|`Ele6_JyZnn(Ya{DdYAm50hhpu@FTQDg%~=KCG+%R8 zd1lTk_@!kwXBB+YeE(_v%A8fOzp;F+UzxKCQVnbU%AD|?*#%zwvV5&y{nPZ7IpIC~ zr}ZmyxWhNYTE8-vI;2{CtzX%7T+IH4wSM(aHIO0Iu-324Ee@*$!&+}LCzNOAW`bWP zA)DXSk%Z1o+#(0RgdcsAIR(o%r#I-AaG_s{(1jkpxmKWGs`+)nk8dV5jr~C1jGt~B zp4oN)zl@7+u zpP4HQerX^7$;Oeoqaa_?<3G7Mp1$dPsoNF$rY-HaD?jPHk5dL$Qv7mq!7n%Odtc0jVZoqFHxh^(XnlC@XI*sN&(-Dvo6(- zVVjZF7k*hDx-7>xtC5-;@XczZy$L5RhE;y zXPp;t?qsFXl`6g&R{JEr8CLuAu^N2Q(z;T`FT=X+1!SrwV*GMC2Xw-ZQnduCRp8cR z{n;E1@J%_})mwbiQkk31v6)TqOQoS^XmGavY#Y;K)mKQ>P3U)a7>aMg1)EWib-kFr zxe*FF?a#LLJhm&k_+_=U{%qS!q#D-xq-n=vyI_Z3RwJEL@XfH!D1cJg2*7zA_d5Ef zBh*|Ij&UC5`XtV;?3c;e`ZG?7G}ZdEn$96L>z!((p>I|y>(AzBaIB_t&{>bwtuacf zhGG1&d|fX_s_JK8E!Fz-Q|r%a96}z}pH0W&n_;a#KlMG)`m;J8AYaqem~vF6Zta6U zVw&*S=DcH_^RRzaIostoeA7|;owcAj6=0vMBSqZ{kcUIiFO{yD;_*#;;&(bDf{WHt zjq=cqriZ*%QC&i*#@S@ArXHl~jEG;_U%t~7D83oiTIM@zneTLGgOh8MhHDwrNM}=| z>WpnWYkV`#IwRtn)y-PaoR)BB(aNW04t!GynZpk5C#szNsCZ^$3)_6x@3Gz1Up5=4 z$sX=lHhQqpGdj!7#*CWm=^l?;rQw&B)Ld=w&0{TJ>-SjtY97`KW-!8SAnjLJ;whcA z!ZT}yXVwbObVq>uLxwepYtP2SfNxqib6mhJB(2f0 zuDJ2d`00up-#k|3V%M6E!8#L4>TGepq(FL=Fke7he%AheoY|$&W4jcJJE2-TvvuH` zme5=caAQg*z6?urJCF>3E`WS z&-$_1NpPc8d!@N4;G5M__Z|4A(ll<^n0i9v3N83fcW4ow_UsihS z(PlA0s&TO%ZH^MSg=>;D-EWQ(NHrOl<}lj{?(LckOoy251gX{vwhlKd%CTJr#4qJy zRup_wNtzV}-?ScPMS*2gC25xi@lB;+mj>}o>tXg3*g|zAnLP#HR2pVa!8fgkxoRAn zWd*;i1lA|XvQmhvcGVS`^4J2F;hwXO46{$*o8@hK$%2?6)#P9r%4`;}^%_50XX1ymc_!9;hUDgf|KE!wuD(PU~SfxFzW@rX$j1Ffo~qGeZj03 z#}<+dzYJ@8sDD~3+GRqRtyKngc>|NHVNHG(AyV zkg7GZAXm6as?xB~R`{lpM0_AOQ(0&$q-qHeFGo{tWY{G)7~PGFt_Esq^j5$&~VJ;L2zlagK8!Z+h=vzEmRJ+{kM_+>J%S<5a5 z;8wAf#q^V1VZxz|?!msdy)J$^>Ef5>X4ggFNvc@*YDzE0H!bN?U612iG0K_-0reaq6^&+)U!O1I9N)sQnV(RDQNQ#i6ja zrR`4f%}S*!K5UAtR3;&{)Sv{q;=_Tm)*J>TCl}k}VjHDgY-fvaS_0e8;+ys++r?tz zr7c5y*TNi~=$iUm4L%h9eRfJ73h$O*wtp=zRuobEU(e4j5bYi5FIMa2SUTa)D)_N< z!k<_0#dOh?=0kz4s7cdwn*|?1s zZB=X9t{LBqi_JFXU4T?=ZSyXGBY`1I+t|Jsn`X5!QgbE^0Tne_-1rA)@^SE zqeF2eO^+_nsOo%l0X7ROtaS>DY=Tr3jYSzim=Y_tiOb>$Ak`{h3kEy5MK99sVKGTy zA<^cvSR?3J%Eh98z*eI9T2xI8jR?MJjm*;(_8r4ojciMTRF$c1MDR_! zrQM>2UyZi3Z8h-ihmzV)J5*ioi{7;05@2qHurBkHb{UwUTt(r?;|hjfP9XT@gpOaX zdidoLP8-2!WeC+2_~uhxLm<+X6Cr+CZn_r0H_Jo!X!vGW)r>HQ8CG>9z8O~60{G@? z{(}V&g#pdzbS;2yMyK5f<~6I4u9oo4YOd>8d{eBQ8p0r_H8*!wnC7(RdRBuCw>^bX&w1iGVv(N860I~sg5tgf_hQ_JXdwT^E_XSKA|It+$x zFyohEY298}wOIE}mM^W3(bz@+*{O(ZOMq`GXuG8gJEj%TI<##BkgAoljld7)e~MpL zB{iDio8_xI6*f?#)2$c2=}@r)Cd{)&XWa>|>9Q0-s+HaZ4Ht2jYIoc*M{8F!4`+C= zDb{YAz=h zpU-}u|9bi#{P_%j{_g+2ydQWmgJAypchJ7xzFY1ecAHaz(1#a`)f$XXXZQEZ-Ts%w z=4MSV5?z5F4@g}c{lYmMy$%guMBv4^5hDSTdkM-?dJ3&?Q~u(HVb%Z zpW=U4v)S`m{E;}nTYg&I@0UBG{&}~3xZ@p);jUO<43pGTswWYj0l0~Ysn#Ulker00 zkmnoJH!Kwcs1zbqm2pI4622rh5#g(to2NYb4@ggrUOnuWUxUUtd76HCxxN1LH2BD^ z^I@~jeRlkQ*{*M_iAkgm&h0N5L?l$blOr$Q(@X6eCr^W~uk`EN-I6|Y)tty*JwKC9 z_;!20&nC`abN2I}6OPo`S#Bru=cOe2KIrbj&>3uaJKo4dd4?!-kAD>3#2dV#bG?TNjuql&oQpx1w3@^JKafdxgQl30P*DIt^zFfKl zL*I`$;pwnE7`88$n@{`Cr$O4;?2=ml^8WdkyZyH~Mbr#n;pZ7?s|&&;31>(+(*znu zQ7)QxG7CP>XuMyD<^#8+2&4_pK0aj~b#k1-AQ_WFE-?^nE;*Dg2JGt7CM+-O2oZ5H zk>#9Cm11*$%yE{ipK_z0ua}Emk~7KnY6%MvwD@08zfsNIJc0CKTcXs`$Nn8bNC6z3 zN`J|vm>6@3UW~K5yJg&}+72}+JWUrB5T{>;C6)|E2yz@=E-<4670Z<}r4c@HO;Va%VTh($;sH@wYKZ&( zw^gc7=dW4lr9_en3B8o=d#zF=1(n4dvEb!Rwgj3+>&n zVd5{`tTNoQT8h>VzhByHAG9dT =R+}NSsEb6|T?1%12nJComBV=yP{dBY}7g&4j z9gNu1H6ZQ|T(FNX7dqtZ4tv zGUA?1x$!b`vCn6BiIs5Fgwp00OYHZy*JCK&mqcQP0O-!@{&K^!(COKW7xII*O13yU zlAEn!qqz^3pctUPZFhfS!$l`a87fR;NlPNjF~}J&%B+wijgzMt#%>trQi8s4^zrKL zZu`G@3Hj&cX1QDJx8cV0){4 z)qPmqbY$zKv;27eJ25CCqt>K9zB{FVt^OvZ^X=w#^@;Y}`E(M?`)_yH3zaA?zub`; zq5naWNdW=Je!VQEOgV{*EXFRDzYG)Cw&dv5NjU{gr1!`-bLq!9%)uv5)1(i&%nkB7O^M+2!TVyeR-R#p6RZho zXrQ}M@m?>!EC*0-8bT2v`hk``Pk2d`X$2FcEPT@<%1VC(ZmpO_QdcC3*ITmHB{}|o zght)V$&xEbE69%nA+co>@^?cRZ5G$&`xvK+aymH@8s6aicFY}1)x4tJ z)(8AKSpjF<{rHoZ!W;p&yeFboaI=H;3rW3VyPV5Z#u?dYUv3vSR&ppP8c+oIdAGR3 z=peZuCzug&TDsd%g=KVDsK_RCM$SU@j3;IGMwk$U>8714yOdEek>rEu_4XLKWX{_n(=~ zfKZPUx#*ngW+W-(Qz}CES@`Z&EaCUt_sg$)&zPV%-5HKU>I&9-NseSG1*q)jOC(p=ze9BwiC(Tgea0#o%%lN} zgq#7!?~oPx&j-o=>8WcKX-cX#8u#+k24$cF0SvSqC1HL@ii8Fck$Tf0HKpU@55`B5 z12tgQ!#oeO3#Q2!JpPj`q+`3FPnEHYvEV;Z&kgN@KBYiRoo8oet(06gAacT^)`9^I zDh)-TazX)4%$AqZl4dw!+kz-dbTr#X_Kp z13FDlT~w0IE*KW%^whQ?mz&jIG`|~f!XY?lptyM*X_R%Ok9aANFd!mrzJQw7N-;eZ zlnSF$0S|R7lWgVpO_a6@L$^+{0ip%f7$vWKUtz_y`=;wFG$-$!&}#>_P?Al48LLxE zP|wnw1IEucf30@g%@<7L$QIKT_xi93d@v76=`vvobrAX6btmr1f509W3l55oQdr}@ zdD!1QzygE4X~;@eh)lufj>9zVR_xT_Jwwz7L;#XFbOITqisF5~xrd_?EqN}s%lp^c z{qwJ61qi9#Fw(=aHu^!w^BHjtIuU4Hc%-ITlqd_#VOj5~1jtJnXt$%S8Og@2^6qE% z(nIp{87#tHE&gQD=E>QT@^-PiU*^-LftcXMZu_N>^Zd1Fho?_zFrI&2UjNBmk>}7s zGY?_GXFV5r2CL^nwuzyoy`sY z>@~~5C1&2#Po+r4)kzp)d)VEtLNy((4+A!8DE3P}k$N9obvtpn ze{;*W6)9Pi)$dWSq2lXw6;AiA4nfC{n0Qhh-B(Mu&BCO_^OoCgp{L`5yk zQ;{AE(_o;X=62~c_)e&?BO+3*Hi7ZjNGSZu*2sL92w4ds&jeToJ#5&%@2WxKll*;x zspi@B_400i?`$_A0a^8_fMm9-rk`zZzMbaKm-inwEBIPx6QxpG+7zCMkp2wBjDV)F@u>>1S=`|B?&tk*OT>?NpP3G0{_;offV9@en7xDyGo^Lie%;;ZnW zj_Y+lT$Vq>cykl=tHCFXN^)-xD1U$Z!wjb$J}Y7@MKRuQ?t`<ot7J-W;SH_jrq}9fFw#^bw!Pzmv8)y2kLT^e6hrR1HtE#&$o9NWeFAm z;t7tGA5QhKH_Y&erdJ|%g+-?hm_=0JN_{w|`-uUR3?X@w=2#?6RY=36oC8w-&>nmK zu;Yn=matvBIhLyz;GLBB6#%hB~ofJJOkCQ(+Zt{4bJ1rsN z*Fm9BSz=p6u&1z@GQ|Fq!@S?pjhI-jN#OvavGHcJ9(Fcojx<~02j&!Z1JZG9b{Hn* zY==WCVE}D1OrlW~+7gEp6)@x;+3ZRTWWZkDzu@~RY$vCGKI~U(Sf%N1%%Fil{IFg( zkCZ^VIHUWx=VDM%ZTa+h)yO7%35Maw9^lnEB4+Oh7~*}&T2-D*pvDE zIj-zUZRCEE%zUzCJkmQ$GrQgf#EWhOZK}Ov$@<--9fLlyHIl_pAsn)CT zQHnv|=ktdmLn7Dx_E^TcfEedSRI<*c^NeDqq$=3zU!tHTV0bfUgqe^IYW?nELtOw} z5C6{As|8fUb6s6P)9@jBl*U$8_I`M^!iAg7Cq}(r>^@YNBHRtV z0AWc6(aV}PY|&(7VY%YjQ5Nr4-ls)AP`4tw?RI^M`JJ8z)EZ5ZP7o?emNIYo{sV6c*v#;_uCA^j#(lJYgVU&Br<|_V{_9~&9u~QZLYxnD zs6|GI0ts9CQ*5cP&R+*-3o0H{ ztPoKjLF3Ehon@o;(qcYBF#iA#C%Of&*i!_8Vxe|BboNSm53f1RBUnaYA%6Y2%$MED z=!QeFWFIzgGhE(WUBj2SjjFMjvgyWY$w=lG%Ug_6qb%nyTk!_M%gy!rfe*sS$@#C5 zDmlve^RMK3B&Ry?w=N1wE|N49;mzi{rAp{OxUDxL93hzFbd_#np@WfcV6k{P>U#gk zq2AzVa97Q!t=~KTa9j9exx4zjxY_=Ff3;lSmf5>zsr1Lo+lGd?ylw9`x3n-lj2!zn z*o@JUXeT|=u9vrGSUAE_7uL%iT_qc#eZIbL=}QsW{0tV@Vm}#>%9bFj0|&IvI{3;K zppEbM9ejT%(;}S9V!gU)=o+GUKERbTQa~du0iiZ)E6_6QhJq+5nc3m>(tyC)f_!nt z^>sjW_8{w@Y)iy+BV8dN{IL0Rv&HJU*u~(F@!#df)&z}oSi~ZD&PdO74&UHQTWhGr zd)A<%T2Y7Cs}&Z~6NWuBzwB4r{`fv3A7v2Tri%qHA_?d^nQnY>o4r8=OthXd7u_+5d+q19ZT6_SlusCFp+T%Ve48h zB~c*TPVza$B>ID}lYw&-^87O&ty)3S%0)#*DPJvN*=o#D$-9d>1a=iwj>~BBZzw2i z5#0XB^WuyxS>(?#Di7N7k--2^7RLmr8x*6X58?~0X zw830K20oG$_vAb$?Y*2SU+c)zvAmclH-5vi|j-wE51WEaa7!;uCjJz&kSp4_V zo6{#x&f(C5^9nL4L?NB1)RFZuLZG!zhWyT6bTc#vII6b+!qM?hii^R=ffDBq-AgRt zZ~ma*86#G8+!~_$HauR{i2Y){fXzCD(jXUuHUdt<6MMJ%!;SGa%m*^{oxgl%cJlEo zlqqb5^H*0N&!e&oj)7^g(3m2BK)CX}xt$94l4z@YiOOo5Cm{a-xL{v>!?OE}yB!LP z_iNm#qYIEB2u;dXbGkQ0U2ng|&D7INF|~Wo4QfF?Mh0uX*(MFHCmCiHcM~Gl^X>Wp zXWJJqU!Jys=%T}N$CvT*fZWKS3<&~nq4(!+v31zU75KNzbx0V0N8njy6HK#Q7Yl0 z8AKFde&uOte%6qa8PYc{(bj5olEKO`mr#b$_KO8741ZmIWA_9alv!^6O5NO9c(`}I zT3qk8-2mAnr3D_#;#p4~Y8JTR(%s8BYh!K+wyUHGwzbT^tYAjkU4NFn=twzb@9M-~ z6$r=8iR}?1y!kY;k@eN~0Y{hS8&2Mpxt>#bN+Q{{?WJ_-N9$d5{=_&y3r=Q6WHgiG1I4-!FMY0zQt&d1zQm=l&)8mZLl%;` zF$!rgHR1l&ay^ERXli-AeD{8P{`T$JZgu0#JXTRIuj?OFwX|Q*w?Ti zwEKyLuLu{fNdSv5NtR=>PUrJAGA#T7->Rs{3_XQ-Z!-6i7O?0gyHDR&Vw3rx;Q&U6ad$OkFyYC=1%xln5{Y;@{7gI$tSHy`lTbMPZESaGmAk zJ}+`@Eo=zq*h2&|{p3+L4HhB)GGZ&S` z%~n;imhwb=J)oyZaQ=YX+uF=qW99O+J}>E5y{M)VuD&jBT$bPXIPLA*d~WBQJ(2j1 zPe%m>QL{}Kw>$)hJ^5Ftr z()rb_OY-9+*k^N^4GvC7n>k0qyLp7OlctF&oM8B4Di;1Y5Q=>rh-8?;0>S5TaoD<) zx_DkJDTx=4*TgZ6i3xX4xZ;Imi4#W!EfC;z$U?b|kX_;=owv?onW-BYU)hwdTF6z} zB~DD*Cr(V;X_%U}*9b*g&q4cP`6`+c6EOCvxGzpy_xPAj572VAW1}T-hDY{}6E*>G zQ|J3|SqRJg>Jq`Rnh#Bf@8=&z_MzdLSf1H|jfH-x~1BS_iE-*sEeIQI9bb^c*ydEhHp2+YZWhPA5_ABT z;K0q}Ww%#`BJ%ux>@A%G${+UGlLH{NX6H&?W!U0X9Z7pfO}ns{PR@mPfK==ruxH7v>r zu_vP6RvXEyi8^u$89~f8+s(Hx+lPDBjCcac)MF$aR~oM0J>nmnZxQ3;dcXdLH|B6> z5jPCky(Uu+cN2O9dA^2MYOaV5s;9wIvK__NGQRr!u*ZED#ABzJoiCS*KSxSQRY%KT zBd7{rV5*S*XT|UBN?+-ropY}lL0tTJbo|qtjC*w1l5>#&10xuA%d3Q)FSjkFIA@nU z5gFO>P#Zfe>iimXufgmZqz&iRByl*imI#uv4QADuk8A2`bMo{AceR^40SFi*^Y)&j{vF%^=Lc(OFi7zq$cibT9@E+TAN&wv{*J5)W-pzHf4&=`ey%`d^71<4A}%i*smNlDT<0nJ3YV1r@!>PgGYgv z1a0@?U9ZGwpy9zGJJ6d|hM5=}aew-X!~LuY2TUl%_a(m{=C89fR#X;o-^>o z6s|8)hG|cvZ-aiQ9|yf~kidQLE2479Q}h83Yy#}1m{{^ zkpUtnIO*lMCw9`Lj@F$Z-eG^j$4rNEUo$-_P0*nj(t z#mN&~g~rW|SBOzSlf$Jv*={rGi~^aB%%w(ne$_&SHE9bO>9ZC(Sz3G>A+?p3uCMrt zt^$$;7O$|ZMr0mAZ1)wv##0X0B`XfxfD-KRM#hM1V`qZ+0tOQ3@M^<%>**M@-hL|bl4qcD z$&gdDSaHWV+5MBH`6UV;^2&Eon2y5cVO64}L1cNHqhRZzIqi;8z<-@yh~gPKC6wI) z4QW;ax$?v$&Lt?zjL`Z(hCe+{vm~ku1ZJ1-xg6=+>n+_De1p)KJGENmNH5OFbW1^? zWV0iu<7t)`XJlJ{xRX6fnu^^vl+%9YVsNC%7iS+4E3A?O2MlKO2enDQ*|JemhPi2l zvLcq{s{`4DcET63-Pd%{8hpmsu!lYZX%fPc?E7GqI+%v9*fEp<&bA9a8S*C&_S<3|uc{$Vt_C-cy;IL|z!=wj{{+BwEPwTo%AD$nta2)bB@mvwTS!}Ko3!9FL)H%RVcYx|uX zQ%me(iAE{+GfV7Z8LWA8BYwvVK5B0(uT5ekMPuKN#S7vf;UQqD=}s5_~g~%>u^sWE!-||3)Uq`@JqJK zTQ$PJ-{IXJ5;BoM2BL})4s2{X)%duUtOKQbdNrq<|rjvxNNpOfGz>rQ`SwepU=c%zJ9^#auQp|4u5=n;#Y1BOA;1gC< zp9dlR5_30IQ-&di$c+Y7tf3JV{cn|6N2YV}h_7*{_~pi)PW{zoNV@(=&Cv1_+ya052|S+To;N)_A$z`(WVnPl`CUHW+)xybQ9(F$5Q`}oq&%?W)Pa&1 zCJ*$*2+1Iaffk8If?LWKCet9*MrX;GGi5m`qUDcJX<{`Nhsd6baWIiE3Z*lA)N^^ z=c>?7p4>GW0kX@)TrMKY)p+eDX%)C=_$L>%kayxoBE=LpXtR`H*m^mNVLOge<;pVN zwzQw|7M;QX8xr|}CTqvJgnn79VL_pVG1r z?dUC4E`qCdE+QK$ono!lQ;KZh?jjqqr^rbs=~6{j?-G;tDwH@pMpc+cU*AiLW0LJ^ z&=Ql$Qn6wjAeER$vsCDOX!3~6f}5m}MK;zx#9FOe8!!v*F0y6YQ{<%EbdN|bdWlJU z6-pc)qbkg!ukR(rG0Ao{Xo<;WsaP=%kV?#>Nh-3%NtbWQvN=so7tOi!bjdtQLks4Z zV6a@y9KKSx6GD+!R>!fchm9Y&dO8zF;5g+|NgCaLQpb6NUi9NxQ`|>BN=;iPgGHYpttE#iu~NxZX~2O>s|uzG&?2}& zE)fXz95@Sd7|*o27(;x>ud`)Wv}(&2S+1NQvjFP(oBxg)W&6P!*m5o6Q5i*6 z4XUGvO$T3*&_y@!T}5>jaj6O`6Ot->g4#RB>u*fSP5{Af2siNP z_L$Oe@+nz3axOf4{OK1`g{%i%VW=fLk&W6?mF^>GOV6rbTY79oe|SPu7@~6vY2eu;uox#(A=$69so@O86DT0dtJ#jvl?i*W@MMRvvO$AbEVQWDY`~* zS<{U(sHnOzaHQ+XgcW612Bz9>gfdolCBi}b4fS0wv{m7$;(QE+=Xp&~cph9E{hrc> zdqLrO%4UVWpSIoU%;T2g`6yr1=OIw^onM{z$7)MI9p58p+H+%mKA9$he^$ol~a#m zEtIZL!L~7JP_(0rS{3dHlf2?B0$LStkwH-rmk|YpTmTH(e?&2NvglRNs_+k5nI>iQ zJP%ONo=K|~@(5Eq6$R}P-3r=6yEH%vHm;x*HRw0Jq8TtSNp&dbVrix;=pxHz1zqGY zsd=(;IzT~d=?+rR1t$FpI%YIcL5ED56to9SRM4J5tAY*?Z3Wb(o9l#1#r8`{P4CPl4KM3_2A6=D%lSBypI_=s(UC5p0u8uXW5(F_=vq{b9y zYiT+ZXe-OIB5ma`sd=(;>QStP()B6WHYN>P^D0SekPY1Kj=VQQzMpgp2nL3?PI1}MSC6||xT{iatm z0|q9k4h3B-&2$A_WZA5siyS63PgYI`C}=I+K?=ISq+dbDj3z4RkV%t*_JD~B+B0ZX z&;g=NK`UU;{_P67#v+OlC_$+Up}l;f59vj*F@1bIz7?%x7Q)NNYuxVg6m&8B3eoVE z7vDCEFRN>WhLa#`175GTH}6+pmhz6k8A8bZz1ZCpXwMhdpO+u*5nc~ZcH;Sgzm{aS zC=g44w^z7M5Bhv5IXom{(5_e4tNlwn-TQ!-pDi}xpr}l4LGEyg!UdEZ625Gqk{}og z4iqg3Pd}UbbYW=2@1`zXDB5hAXO;_Y!2x+C5~+w%FYOC2ilTRV`fGA;o`MQ|clwmy zz|C|v0cE2EhFrt=fXS0`A(?eL4P$R_yxqUsZY9=Sfqrv~hNZ`=CH?+s&}0Ee@1>T)NmH+Kcd60HgV5P$~2@!v!zTq%Ei=XPaTl|Q zum?)!*uNo)s|S}5)}tWR}d=_ehp&rMS@h+kR&sG%e_R) zVF(;y@_cX~BnhS20_Hb{Njvip>q1N@s^!*GQWWYhL}I9}cd+0f%}`6M$%x(JU^O+q?bh%j&=W9iLHpbcr{c^|yLaK=^L?;KJ6|SiW1{ zEN`KlNR`Rk!rZGeL(%jn_#=?j1;Lg#t(cS{%WJk-tiSzt+1Mu9-7i+Rx62)x7kaJ_ z%46}j)#hgVcSDo9{6f%#{u{!rFV`C}lewXLN+D`L*9#K$pCd`9ff7wR9aQP^eQYp2 zoE&|;!WGs3#hbq8+s*Cj6GaS)g6UV%NAT#*8ro`vDvFPj(v6iQOe^3bRYP z)M`rww4wBiZB?Mpw^m5p(q1ur(qcZbq0KyTU>C7g3sMX1rlYpfY8kSN(j$DAEhe?)044!+!AxEE3B<9zMNa>^>n-xtmCSUT&7V#eTb!xkTbRQne`L(<#S5 zkY5t?{T-f_-+pnGxpcb$`gj_qTvDVu3{(0>PlSApQyG`+b$C{Dg`sWaX?7DG7uwPJ zoes^TZY?q3XX`a0(ye|M@nD_NmdE_+Zs{I+%oN{>6}OaYBv}}g^Z9uGJMo-j^|^Rh zyH$a%orq{vt?HOF7jwCGR|WI$T%m(*k0^L3_!s`NE8SD>Ng ztEHqluS)S4 z0Xs~!VM>{!Lury*h17bmlFu*q%P$Q*h@Z~cVWUJvrAM`v`Uh6U)pCCvh5xqOZa!Ts z_KO4g%-xf<+5#UgFRC1WKPP=#lNR1HTdvoudpsUQFJcqsihfstI+8*La}EGH3MK0e zDS0j(OWd;TK8AUr{IaXNfUUlRX`XVo4-?u%G2}Or@I#DM|MZQqpdnBw0UoryHf@NN$<# zOKS>R!d(=)mh=qkZCHz-xA1^Zy2btWVRwxQKEEo%OF}-~cMG0dG|frY2yNXHQ6)<( z1tVjlS1B8(fs`#}DCH<=DCK;zNZF+gq#Q|#NDD&Jq(nX}7fG`!vPcq9BqT2ut2NG1 z_YXU~(B&qM9Lf1^aWC zChFZ0kpSCHkNC(GdqOIsa!SsS0-mI_XnNwI;Eq4JuLd;z`8r1RY{NpP+1go?)*>0H zK?z4lldhA3E|z<&XD@JGvtQnDqMv(GS}JToL{H{s9x;0U(BaKC-z)% zS;gEeZ_1Uryx&sff8xjtpzJ>FaP&lbe#$cC%761_%^~Z&=?Ip!NRA($Q_ykf>Y9&b z?pLy!Nb}9m;Z;;J(aWK2wy^Y2;w$XeS9dN@f66r^0K2;fE@-|4S1=iuRvWDd3tg(R{v8g z0^-|qTekwKMzGMzYKXEp?Lw07I+?XY6Tq_$&8(lbZ}GBmn&G%7Hc-=s!LX>99P#{_jYoL0(NK^lHmsN*JAzP9XU$muQ8Eh z5#V0aImda2o8k(!6eEBw0SZ)ZW{OkO87fY-X~MZ0CIybWv6z^~NO87wzoXUgxf2Q$e;&DSqANe|b{l2$(k#tvRx zzFJ)GU?{s=@Qd**?+oeN-Qv@i#Vah4;mE7$HU1)PrH;-hNen(Am2sSsinov&vK$xc ze|{?r14;0Lq~)!uElRUKE;XZ1N#~(YX>IgdT93hZs=6%r*emL+#n;+q-YHB5>>5*pwY7@>1Ij;dI!Y z1^I*+AaC7&%VMIm7&cq47WY|DkP<^KUcSr%?0Za(0%ytXdrZztpe!Iri6Q4#S$h2( z6I0jElKX@hU`FdKDoGXS$-BxJI3122UirVKu`#{~bO$EL!us_nH=|$rx1J@+U#%2zwIr+4KS*PltX# z9s0v`=yc$%Vad~|yHp9wH4a5tK)9O&eo zd5UeM6ZCs77ys@h%fP!&W|$|mqx=T<%-l*K!&YuTh_wqFzFJ!dIVnEcYxA}^!?bmi zTrQSq&RS`RipnDSb<~=G^W=K5#*{daP4S}McxC6c(7nCID)BqGe940plgs7!98w z-t?~Spo1K6gfZk5Ts4~OcAf51L6NzOmV9zFQkqc?bZ&Q{CxDYV{kDU3k|lH)i>}h_ zKFLHXrbs%8Tkb~0VC4SpRRiYX3k+c}hta_tS5S0AvFYh0;WxLUe(vWsNqbu#&E@Zw zw`&|cvRm{dPS4j%Di;^%P=>e@hwCJd!s>Nr%)E8smP&ClvOnVN4pO}#OPuZPRM8g=nLdgx<0hW8|jAE-xPe*_wP@x*abqB9* z2}>c}%zdrZN5Q!L!Z9s+!0O5K$Q6=#gt{p;IR1NGsRVgvlRG=p51ON`^A*MWcKtwZ z15^~AP2Jzc5O8`>btKbvr;OkLBv|DFgM<7W2OxO2}OF{_~EucR1m@ zq_fVwYS@N}RW1pA)fj_D65`@1c!G|}!E z0kW{fo4=-bOG8P229FopU9`5d+0Eq*8MMo}1kp^0`*sb6a3FZ`}+JthSw>#rrLG3KUxB)s-yQ z+@T;f6`4PFt3MvF1{vuLSh~xtBWt)(8xT1SU|{TM;*~qyvwK78h!v8NOh=gv268Jq z@25N#oopyigfeLiR@Q)(*}_jdV^V-W4gAkGvK;<(`R$%&tX8WPDMr09R#B2dDbD;? zyXzMHD3C!@jOx@BldKHth4l833N0p)@XEKH<&OxPA=X3KHH9?W*r}!5xg$H&sk#XM z2IIsXX%vZlJNw9~aC{#2jBnTIRLRhQsY8`}d1}f5D@KuvACHcIn!n#}*DvV?F-2K` ztKxR&th+6W;Nt9jyW2h7iN&mz-`UG8Zr$rzteINY{Zgtb=xTu&lgr)hc8A*l*D|q* zlH-5NZGCa}5y9$ExSs6zrVGvaJUTcBxVn=i)j8pe!zc+|M}(~4kyq#7uzLVk6=(h$2yJAqB*P#U0N8F)zA#P zZ^!L++uc1J38{Z#N4@^`Zb?y!mN#>rzPY`c<~ojzmghu~S34Me#p;4OSlfOgxz9Mh zXHAG(CXxigU81O7I8r6`b60lSYPhb=9VORtU-!F(1S-{-n=%3^r?vtP7|>ZN7iBbP z=b#huwbR0#(me6l9n~QsidNGU10}=b>%$#)HL9aXxuWZE<3wf|QE$OUfo}Y}Ifhy% z(>N66hHt$iBCieOZ|BY(-6l^b>Qe}6Kb%z-toEDVomc6 zw&3kYd0{+_j({A4UfXxwu>>Ev(Kv+Ay$@SLSl1n$f&S;}&Fkm)*9&(Gq&(+k7GyGw zgZ&%aShov?+|6p|rEUda(W$E|f|sjLpJi1)Kj-C~+mCW?Y#l1ECIT$R+BFQOyp`>Q zN(`>JFScjw^896mCfZ$pcKfsUM>g?bgEu1e8$fl?}f6w4qZ}mlJp1}e54+tQ} zu~oS$qSC^ukV1utC})`o+A=gNbrwcUHPHy+6<@7AM(_9z#v=^^Jt!Ez@|%S&cfs%& zcUgG3UGQC4cns0*_kMOSeg*j@VdSr$nRH%mKjB#E6J6f=gRU=y>(TR>MvWenxgKkO z2wD8eR>3F>7yd5q!CLbEJHUUUjpY>AfGXz+J_1#qz0Lw9hUR_XH#MyeByB7oQ=(Sb zN0g}XoGel2Tlg4#Gi7y1JyTwX*3SfWKu>XHGFFdJKjSr}emRfX1KY9Aa_Wc}ozHgnjkcua zB6dWa8D_TKkmZ4v8Ios39NtpKXWWp8+ZN&6q+G%wN_G!ZK%Q4meq%4+41}VVpXdMg zuyF1c(Z;1{m+XJVVvu*^iyS#z_p=*$CJz<~wcS~ScTiuvozeuwGFpRPa}YD+dEVSl)Q|yi=0P{tnmP z)jdxUdS!y>ny)l?XMWae zkdua&#vS?PQz|BrE;Q5y`=ch;-_((w%+6url( z?$_+;4F`m)sUs}zEPfex&?UW)vfNycn%1vAKkV@i6C8`a(5h*<5cjyGgXS-WiD$1a zhPdLD6R#NI7K%+kRYH0~51}J1{_&RE_~;(BUqc6t7SJSockQS0n? z6I0MF0q)CHQDF^_#=6Q<4i%ZnnqoeWJR&+oz~2prX9rk5(v9GRO;^gIcdzB}wpa6w z-fGZHIZUsF6{5Eiq<6TRkGg3aRYw;h4A0g{5Trq8U_$oi{X|}N@Y&*LezSC5O}B7W zbwMX4?kKPsCZ`k4C7gTcw0G)X7Jp!M^tFN0QdPGb0#Yw-3^25cfiH^_9 zNI2C_$-4~M3pI0YA|y}eSNsMD$5Ok;{^m}$4okYzf~zsy-0lFCw|XP)%Ul;?`50{A zwm}w3&c8ByOTG?(LhiACX~wZWs zQTzueJcSML&eo4POsZqcFhU@@Mly1ghq;gYB2eRVAqX6OQt~=->9%B?QG-?tZJkY3 zS&>VtGgG$#?)6ef3;FY#PfIe_h=ICeT44vqrX0u@b|jL1cD)4`*f!ovz7c^2CPi6o zF0ggrE7P*yO!mli2a3(2S-nQfr zjrbM-whXV9c%PFC)xEY?bRzdh;FGk%6pnCZ#MOcNW{|dZq(*p-Y|=Y~ST-#b(_7Xp zN~_M zp^HD<91RPv*f!QOtO!9at0Jd{Ed1$t3wxn%q11IO#`v{tTW{p)o{=L8>VUxU1t^>- z!Ei6tIFJXcJu{O$7PF*F9mw`@ZU@)W7$jYMLe1tEt6Mx-0UeAE8;baTNLrb;5 zdmQWKn&((ihhwoexGRPbG*BP(ehK?4XVg8d59QP};^Z90trBte(gF&OCF^SlIxXiH zD_b~UE#&~v--c<>Msk8iTOw+J>ek4}li||{B-Jimfg`Jo&(QvGFUhQB3d@0$4&B6t zo-&d5NSdA($?knR9yO`q8(u4^v4Ejfmuk`TAeF}0FYrwIx6K02Gr&3_(}YY;<3Q?G zExlI*{xs44y+wzUGE{rsdm&ExPzUV8X0s%(9A5Xs1JRvm*UUw{!)vKc^a4){Kx4Gf z{6Y0-8o5zhx8$O0;TGD#TO!#`_@5Pr6u@<~FALc#G`R&(^&;q;q6Vy`+JpUW=1Lhc z=$K7PFUk(;#4XIRAZl+l4xuojcE+(oHxWo*6g$4V_8LJ9SmVl7N4yHw9K}jsT*s4G z3+!?vJhQG62alL@q<~BlC>M9}o7=9#2BmO*-fMaBK%o&AVd#ztytv`Wl0CUH;icK; zXXFQ6LF)#05fQHH_{rsR!iZcf#QkW3m~zazn~#qwDIz3sLo7fE59wGxF1O!y6uJbV zD;}5a+&LGi1V3C}^k$$6^)X^ljPfGW1{bbLy#}mC-tZc9+hzyQ-|b;QF!9E=u zc2(fHB2pXNjmpbB1yitw!$XNVZZP8Q`r{ZlFigiV&~{$WH!Rs-8}1)F`i^TN4nZ%g zd81nK48T#-x}{yo1RxHIY=^YPU0@E;iW#kMLQq3=;V zf@oDdLfxr&Jg8OiWDsqN#{(xT9?zmx@d#qN;)(GGDxR3xp^B%@twZsAH&OA_aXH0P zN5zUq=(&eJ!cs-JO&j!r?HR;^XdT2t-8qOosC5u$5N(6l11Aq+&!Tk@3u5{pj`0T$ z;+WZ?gSgJEV-P=^IEd@G+#s%_;z2C*(}ta3dxo7LT8EubcMdxbY8`eNMBA|Qz}7J% zi0NY{#veFlVrGYqnL4+QF>^d|%+ztYF;hpyV@BwwjTyoAj2S_+jv1lu95Wu&I%YD6 zwlU*@tz$+I)5lDVKXA;%%nlthb#5JF=KG0brjE;vnK~*SGeSRY%m}t;%m|`&%m{Vo znDL<2F_S^GjTsMY9W#QMK4xP4fnz3ScIcR?bL$v0KTI4mbzE-D)KT%65&CIkMzB3& zMi8xIMyNZ-j0d%jnGB+B%y?kym=VPEF%#nt95er%*fHz4T*s`V;*Kfw(>kVLdpf2d zT05ptcXmtIx6lIG zY3C<=P^58^o*W(VYRO-p%lRIPP5UH;KAjGgqr6GEKAWEF@pS0-)1g01ht5vhu}5@| znEKM`>A#-OW~bTL>s?ExwDFj83PB=bxR7*?X>LeIG*`S7EcVsyVx1(&$Jw8%OS87X@nkQGKkN`;?LqyW}l ztN@fL8hpFC>xTEP%c>=e8#q_@sW+STH$3=(S1DN`mxAg`%ZziN>#F3sbr`1x2d!FR zm_=7`D+)E9Mm zoMBg}rEym6A(ygzEOb*({HO*_k0328+TqeNR=v`~jUz;8*{|?6CJpyqJ+u ztyiJi9z!2Iwj>hv{Hk$^Qvf0rl*gs#n_e_-!3-}`3F?umN^))~hF&PC(DKrm+eDU{ zQuj;{y+DG_S(8zL6JJ{e3_M)nGN5R1GoE+`Yx$)4j#EeG8Iry zqc)@17?rjlhkEi7&o{?`;%&cg5s0<##zg?sn|oXp{Udu5Hv>xUDz(D1{&a5dBolH& z8)TU@x)S>y6>-rk?tJ4y9@A{?5fLA|nK}tAHv$hW18(75?J34uR~epf{({cH`=c^V zv@}Cp*>>HO^c@;~oH16-!hq)!9K$ZcQ zx3nOL!}f%DTG>a=)XZn(9s+BgF>l~tnl5FlyNUn&R_=@E^n|d?Pc7ykexYG`V{1-hs_iM&>aQ+!Q3wtbDjlvQa zzRjx@0B9~z-Y{w5x}fd@akYFHKH3+FdbOEua*QcDrE%yOI)u~Z)Jyz+SbXAW9c}%I z(AbYIpKor^IixPt5Hn5`gyon1ko|(AbaI|*E*R-zgv6Ec{py;{1bEz?k+mK?gdr-= z{~|1$L;-2z@i_N*WHiFPe#g05!xbWjJBhi4E=`UPczJrgoZTDJJsnwH3+pwiJd?qw zqK6tNq%V5nEUOk=hP-Ovi#rs&o-5_G>ll1W=A=<%RZvyM7P}D?N?t`|Me2-S=(ITK?b$zvSML2bXWNgwuG}5 zDo(Dx3tL|s)u+!l_Yb=z#_`2=c@GmAKW!bomSb;@W@g-xNA5XV6gN^+e;eV{dQn-V zpS(|B4&Ra`AK!B&yjzrP0vX@=@Ei61<|nIVry01xb1m?7@Y+3hR+fBQ0S64|PAC@z zbCTOkv~Z&R!~A@w@2-tkm5JXi_YXTfkd5;_qOKqidCX7P!NYLQ`OXJeDl5|$tFI4t z+|{U#vSO-et0`~AOV^tER{RLM`>lA#tNX2ZhnsfCniqCq*o$dz#k(@#C8z}^Tby?B z+7=@Cw8N%TduC<9>te3({#L_ERUp&eeV5WsbLwD$xp^hHSWb7;N9LK}z3+~9-?g0f zetP@6?*nMpyYEA^Je%J5?t2ODUtO1#Z>TggXSjjT+#by{c*a`j00sx(C%yYVkO{Wa zcUwfku!T!|Wn@1dEgDE4R~h(s+`0CZu|*3gHS{*W`>xe&z5dL@iB=78mIx1;WfCjT z4yCubCq3~ag>GXx<(RX|dBS;Tm1pn4XNjSmr26G5xxn}*{Q4UyX8D*BHO`MHQR6vT zqVC~3UdU0ZMUc(L=EZrTItQceW7q*XC3Go*CI zS9pcqTRix94#s)6zbL$gUP8&9U<$~Wpu8ILG+N5W2h?40@0Pm-UIglRKwW?c2ag_( zXrFWx>0{^j*g3(1ykS{Bow!3DP`C2GTlPJm?tpM8UFPkh$ph*EJsB@VX$_Et2h<&y zgTJ~X&#nj59e%g?^kw1V*yx!e0!y|kVG-Sz`(Qr;@B;;Au{)l-dTLL#fVxHjLW8~e1cDUw8%r#qYHR^rDTsXv! zkxMsiIoynKT-`R+@U5tV(ze5PpsKY$5(%-b5_nF{8I&lXnMrlWu*^# z>mep=eEpA@kMla<5%UV;-bc(6c78?QaZ04GkIy?FF;DVVHbWP+uClbng5~wG*l^n( zF?WSM;1TmYhY1^>L-%tIZ~GsEj>xkEI$k9=s!Ye%T;|IK_PqL~b+n#jxS#>_4 z901#2Mh-#xgfX{5wK{OWajyyW3PMA;az33ioMMwar*i2HL2v5#fX9iw$n_E(nS{*c zTs>|ZRJ-C6VN60M!?=Nmi!4A2b?DVcE^31}gniAiM(KX+cZg2v-My^PQg}Xz6ee(N zd%@S&g(m870q+xYvSs*73-KYa9!MS?;g!1=3i|bp1c3TaP+2+rhOo28WmBpvPz<(H~kCI?{pJ zk(;Pn{dc^l>I*H#JX&v2GgsGLMxNBP5lA|rSsQvgW0F~m7R!MrsG#5})ZX`00~z~A zS@o?v7DDvIW%*>48YUWPYMW5W95e2;J-i<2!A@7rQEn3+>}+9s(1V?lfo^*ad$6;CSKmFj zpjbWb9?kLKIv?zm`aRNvo!?D-u(JV2(FT$SI~!2ZMH5Fa_jx+sm~_|<&p3ym`LQv+ zYYx$LG7$h9w|WktnzehH_)fabN$<<74Y*uG*K>;-TIi=Wv|xK0S`a-Aok6rUv$se->!?`RLf<27 zLG%bagJ=`B2et}Z9{1=Mb{&@!b{!Q9Tj+a)Er=drXAo_|_P|zQ%YzsF!mi_T!mguY zVGDha=rV{l(RpC2=;Sqse$mx&InmWovFL=pM|2rPo9H~SRdn*cLci$hxSZ(fs91DD z-y^yVqD^!j*eW`<;tbx0Xn9&f7IXvXQ?{>+2LvoY#mqAaDh~gXpyKF13EJ|sM4_%N zPfHY_O;1ZW?C5h5O4ImUgpyO&DnBZrRMg*{BSrOz2E`HLTj}ZP9^2&+ zkJvnE@N7c%g^ogz0$6*o0#K%Ctxq&aiNaeaAo;I`QM0pRgt>j%s=LOqL5|H!()1fVFkI|PVcv%F~vS6^dt^>BTSKw{2Gs}%XK z#6`W}O%3%rW z*rJcDhaykTCLbVN=^6usJ8~cLn^~7GODJ_$Q0@TqxIwucV~RL!VZ~a4atAn7m@mPj zF(`L{#;T<+D7RuCCBC&|hHz|{;rfFtE4&_LIihR}vg{x|VT>Il7sj}F1xhdnMI&}B z+M*FFaa%Ovl)5QAu!1H;BUWfnG-8KYQsiuN<7@O$RP4lP#11|&8nK{9qWU(+2d;8z z^^0(fC}3{jn;krUFpRMqK*YV?onX)XLe@uMDX#)Km~PB~Je0a(>`GC(0@w;?t&H_H%E|0Xpa3UIg}z*PKvMC!*a^zy;|7b zU3`h5nFabS7Hupav-Vcl$+b@d5gbtKl+~eSvdSG^CSx_FOf+vjO6*EMM(Ql5_L5P~ zk5zjU=d_l|l`&@dn6)>~lWU)L>;r0@vO2U(R=LB=WURW&G$kCOjD;Ro2VUq*3x_C~ z`DLpH4s^?nLL`L}s=*#H%!)z&b9v$HX|$A$;SF7J`@5-O4~fucEbksGLZ9c@7@^Mqf5B=3!zQlL-gA`@n)LQ|b?uLsdMtwVh<}oAW5hot z)*yA=QSMR(T~Y2lxHrmO2H76vt^jV1a+ly;v^v%X7HqGq48r|MZ8aks#u z$r5(d%#wvCk{oJMoHvKZ?Wc7Dchy1$g5JL9}1 z=mX=tDYvFLZ%X{IyI!`%d2@&Z;=JWK^e&9tV$T*4~*jmxVS&CwHzNflqS%x|^d9icok^Sh@$xm5zEIBV=2`a%!xU zkWEv_whW-nyO5XTECM;L=T1;DhXv%m)sxn0V$mJzEvP0|`xah_jNQt0MLM*G7xJ#h&GQ}sl}@=_u$tx>TORG2wwCcTHvP7u2>NCw>mRtJS)ie%*F za%ia*yTy9FT(b({7OO5ju}t$26r=;%5x~h;SG@LU%|HX8x^^&fFQZ1F4`S3C2`MP7 zXVMo5Da!()ZjOW$G3beejm>OLu=F%TFg7;%uh{K`_#YLGCT`1+SU#{1y z`z4-fof2Ir=5|PQp@frI;39X~j_5*lX58sIwg?|b;{iP`w+YdOq{6zoJ1qE5120;U zwkwwAD?^=8e59a{6vgM+#3()uxXg9322|u8<>=YYv2L;+IS0wB<(MgZ&7epPG6GR;q3qt6lYVja0dQ3mdAld{Wflc}a zQOD&3QAfps@L^Y4dqf7&CI|^S(l3ZQE+>dODi(wfiP9>F45CdC5)hGRNlZ1N+Vjkq1!>N!3S9QP!($aRaO(5)9cS3?9g*cT$xow!(<4;Tiq>1j)>s8n4R1n5fP?W^1p$B+ z6|eCbUyctDV)Tl?UUy~ux%--PRmp*y;{ya>y@byh>L1w~`2i?%+;vsmfx2&$2RI#K z9XHAq{V2D>9kZ#o!WCY>6&_Kx-3oV*o?GDpxoEqdt;B8Dvs3D(>)8tGxnu1x@H6kf zW3AYUcdQ+J;vH*2KgQ)^;na1xSRhj`7d!N%%f*77cDY!vaznT?Gt zhvk%IZ|MfDT}kd;|CSiqg1W~id%X5B%g3y}ah_cJ>DLaEI!|{<_l$DaKdphMwMU)h z)b<$VJf%Hy46T75ul6y^$E>|^o?QFK=#4DKgL~tYo2$~1J8rHz0G=GfB`j=lzbrO4 z6sJl0JWoiKSz>r6bo}P312-DBNc^&A^-t{UKxZmtTy`ps2O*Wmb}#O~VX>uvKj%nY*q?q~tr ze0Ma#cNSV4Fli;qF`CIHUp|9pOLvTDx`!!6)a!r}BWkv}!7bDE_8Z*vHgdnj z*8%r1D~x;ZVJ7Uf@{t>vGjeVJ7gkhvdt$ZSly&FTw&^aUX5D@l(jk(&kiAp)p%X22 zzFv_Kh7HP{Hw>l5gk~@M$GDuAu!QmQPSoS_rsuLn$fnJ`iif3C#Q{2w_81r5cI&vW z78Et)&~abQbLhMj7698Xg?W%()SA-W6q`6xcW5t?xpC>dL0rOHMbkzZahZo~`Y$K>inY2a+K`=oRmf+W@US~Du5-8C>G8}u zZhZtGf03g1Hb$07-))R63#h>6+ZYjpp4%86*l-(TK#Vchb*(}e*v$iO-pZ|m1k!q0 zL121MVCrRsm`nF%g&2O=Wrdj6p${3y+zz>FkZ?NeszIGu?{wF^YwC2Dmy$k`yZrG) zclicf#yhA16}bmF`e=E^%~0HCT#@M71W|WK z&kLfC%L$^6iUpBzWv}#z45CdCGSIp`e2?q6oFM9`SP()VwQofb8AO{P>h9?I9#O~T z1W`xDg2=eCR|JtkvWv%C94dgEvt+>b|tF} zqAjb8!*eC845BToj7xDPs|=zoE8Z@h95p%^*GFHzi@RL9oQ3Oo#T}gUXmu7A=Kd(( z2Y3{ZfSjdJFs&Y)j{0HcE-_a)4{v$V10nC6%txRnn1ndp!oY30{wieiW}q$84(Cqr%zivCWs! zL3(@{1=8rtsKjl)j45@KFQbBbd>I{PImMSzv3nfTh>&l%t2g*>Ip{<`MnQLZ9|~lu z_n|{i@;(&oH19*fPVznsp-tY03Yh49Sb{fuAJ!SLLYMH8r&v_ae$Ol+!Z$;hzM_0X z`eyG#Eh;@K(%^k4xyuN4=!i1zE>l8DYZcsOMnS#=<%OK5>2#No zoFyKO_z>OG(_A<2w-;x=gO%geIQ(Neq6o{n*M_HSusBfL#YgwmBhhJ>GNFU14_w(|Ww;sN0YC0zh-XmJrh& zuq6j>J`xPT2OJ3sm(HU+2bzAAmu0{MtnDZ-#P=TM)v?7*pNEbg&d>-SWP3JK}KIuG0pdIHi z4lwCFMzF(^j`8!D5{h+b>4mLPkS{^*Jf;L~e0W#j1KrNYd|^1i?kT{jNmu|5eGgWO zXb+BdT!76W+mBKT;O3*01mAp=BDlsol}g-x5FiNJR=Fl^xaR3GQw{<=`q2&o5|-#| zlkPNzY?=-NGJuY)1PQ(KAi!s+cKIo-rr2JazR&#to4#k#Z;$ubN3h3xW{viE4@j=E zbt^+^`Gm(5v$!u%hcP<Re_+iH3 znAf4k;h38{6pjwCJD!xOGi%pfKXhAabBHG0{6n{;Hiu}-Dr0wLnG`LxKV%SXS!L|5 zm8>#|wyZLC*Gg6yL|azU$2yF{KV%SXS!L|5m0D#GZCPdPu9d7Zh_Z3evxhtVq zXOt{=0-9o+${^c~Qw4CdaVo(#8>a-58LA=1 zO+yW)w;{IIh~qglnms(A!^#kVyp;iBZQk8S40?9=0-U@#kKu=H&SPE&Zl~+a+I!JI zySLLhMAInwXZLnGhiJ#|wyZLn<4RT; zL|azUojZ2!|I8rTvdV0ZE49k(aVv<-R<(lY+oMVyy7#DcSZ0q}2Ss~SA@AFx3Z`|B z>ZsfIr~#mVj~ajv*rN)U&ONFFP2Qu1_})Eg9b4R^I_TqUCLJ@}R78}Aw#(uInYy(T z^wh1LLr>b;33k%fE`&C1?G(_swG;A5TRVYvZ0#Ii($-F}y<0m$P1&VoknOv)0=RjX zmf)LrX@YCqr73ayE=>@9yEKn!+@)mzZL>emvlDh{K11E5_3pGhhsK?j2XyST1T=ZS z6T=VN@5H=CbB?-wITrx>mvaI5faRQU>0Hh^5HII?mt%EoaXIFokFy+e%zBn%0-3s? za_C75D#1=#P=(N@1(gCC7gRz%X+b5>js=wfdKX24YFs4gIo|_1=6nH7p7Ue)VRL@W>%cj`&a8dT7mG~u?5gCQvuhoenO*CkXm%BH zcD3!F{sQm;)1PqZocK%M{Q!mkIf# zxlEuPbD01dXC)!-o0UAKaaPIznr0=B>ztK5aO$k&IW*2n0+>9(#PGu=n3&gr6HJ|1 z%LFrj_x^l-PH&%I(c$JtyxI2;{&tC15ziL)OZn^S^TYmT`}gJ_^ZEJLuP=6sFG~@{ z?iinbzH)ysqrr#y^IwSSetG?{!&8g%YszkZz1{w~TK?n7!{+bR=H|)C$?7%bSBnj9XhJxrr&qJt z^I7~cF7*_&W`1(?>S4e9`pb5GV@*NX@mFN-#oIt>OrH*!21Pz8Bwyd%eRVycK(V^} z?tZ`6FVRN|6jp>IlH$+1?Ze&a>E*D4WfBegG=RQJq1i-*!W+*NxXb&MM3=N)I!0n_ zw)guipZV*cLEes2DYKLL&q;mWKA-)5888~5Fecxf{(DBa`LFbE2p*;=2m|Wna`S2b z`E(}l*3nQun_UK7VT?rCT;9uTyb+W6>wwD$g)%`#VJT)*nTvo$F}=Xx3=Gaj82I#n z!Sl^cSO`z!6dwvvBvCdevp^0rQVkdLU&Hn&r7Hu<wSJk=;OZ(bZ{7t$HSpMq+h&E~Ito;qWdDU8XhDw8sW zGI@cC)3+PVo79a-;zvh;%xh#JyURv?qjVv|mkV6S4C*RnltLzDtdXW{N6X-vg6d9u zM`@KYp5J`8ykG7RwAB}xW{cl1BUM|aQ-)BbY1LCzn@=|eM<-8ayWQg3=|>~Xt&m2k z8kh~ca5-a7+rT&+Ksq|gtV8Ms=CP(LvlTjH_;gfSTFP;%aB(uqE?lFG5^C`-uol@p zT<@`r%eQ>E$ZaE5)+e(PPg>(y2aCnpgVBusSbkbQ&0B z3!@p)rg~S*Dr}YpN@YssfF_=GLS;F-yBn+D?=*m|u>&34jnfQIj!W{Vesx44)$wnT zQ#+tFi%@gu2(FYWglINc=$tR^7T4IO78(T;{;xrwZ3hHdT&@1QY-Zm;tgLn6n{N$T zsstr(BUaYfzWkjnGEa{I+d84LoUPZ}YwX?%Y-u?h=!r-vO@V5*7t2L0+ zs~MxKkl}lOxp}|+bGbRi8MFN0E&XuC-9Q|${%yPa6E*`lnw<_FZ93x>y$$)WwU+x% zpk{gUbdXJAwkZ-Ai_zz${KC9K7EHwzN;VdJV0+W6#T{9ea&_?9Dww+~lv<^m{dV_6 zJT%6cpC11*vl@1yqRhm(qc^j5RFs+ch4g0Dj*2o9FZ15a+EGzv;%DBQSvxAqOgznd zGiyghnTfA?Z)WYNC^PYN>dmYj6=fzzti74FqoT~j+}@j6J4!Rd-ctR*9#%J}c82S` z!nV#pUR=aY#%lpijxLt8ek7x|+iKEuNT)4{0k}ff#Bly=w6h+~TxpdtCR8p}TnZ0vOAfj6l|*fXkRhX z;?B_wC}$z7zK$$P0L=nTcL}}l_08?6adlmZS`T)|s6xTIc>+>16{A3!A6pTl+=8Lr z0~ybTP4a!&?T724Q(q0 z8f8QQp)(}eiL8mXnXyh%PPkR(N?*xJa>6GEg@lX&&vrL^-*6X>5l{Fus?VeG3kryZ)ZKEo7r!B#Sf0#&JC47R&a!=RtgvFjCUB7T zyXEZ~k5{g?o6F7Z_T#CnM?T=Zld_CcPm`mzIJE^EWX!JyN^W$zN>%1h>3QBbUBZx@ zoLm)%DV;C{`W8(`+zLcxR#(i($`^}2;h3Qf7{k}&%Uw{w)I^t04;Un6r%`0xBCctT zGGL0c{r&!10HEd{VckX_XTYwMHm+zk_H9pDMM+6NLw^16AbUu4zq*ccXwQ5CILP|q z+h*})a^6$Wn)%6*))D3|l5lL;&Zh=T=sdwaxv_i*EmPTCf^~MDkCrmBC}mic%}+(* zh_%{}iqvlU<8oAqBi3pyYGPd_R@OL)6^0+=lsr9#%_JK1$vF;?-)~Pp((}VY7>qKS z*nC2uEKYiHmLaz-M|${Wx$uryA@30qWkV~k?PiZKCBH20KXZu(JcnzwMucVp-H&|s zusna7mx=Mt$H>)I^@LXXASt>oL-R zEyo@IWraf43S6U1nO^-r-rjaUZW~!1&c*lwEXzrDKh4Opoz;KiSH(_tzvwrS#$4df*iN#?2MnHZN_HAKVOGmYd+mQ+#W=7 zE(8|3AqDvLyaFa9!kq|XpmY56r_9rosF19?{;qwYwNxrh zNQAo*(joY-V|tBUE2@tSq@zs(6IRz`ZZHTPpA@Xdg@kGEz&Ka+sN^VQ^mj z9-2mc?Zk_akS0DaGdWo>;g`Y!-HY3M_hWh8LcsIg_Wt$m@Z!&IjXNhlbGG0H6?h&( z3)80S=J;_XGh1P0^G@*70{R0DmrcPAT1P4Y_!N`s2DPT6tL2&?F=_X{HvMU&QAuUK zQW`kcof4jus04;@C$nP}Lrtv)5-7#nw}EA9>jsigN`%(BttfhT(xv!9pGc5uhe*Ie zZ%B}E{H3mFW5s?T;H#f^o2$-nhgHt|ll$Sh!lLPR7kil#=Hu~fqBvsDW-5&7vpD0Z zKa(YO;!L7(f6k;!T|1Mg(95$JYaKqHboFZUM?cXhPcGb(e|c|H!Lfg-=t7T*xY&n8 zWa>2{c-&6}B=rmtT<8lCRXg__aOGaXalb8}+-_=`pItWe$X!g3tWT)W*f9icp=rRD^@-XNe860Yl_L}qUaxd;%R=rUipkZ9 zTZ_PT-HU%7)oy$*!Zj`Im)&7=!&R0oBBf%K_-O&LH z4=s0xGmRRC$^59ML6NFjRVU2i{n}~P6gsALJD>b_{;vDZJ7e+37YoV^EZpj2Fe~Rp z{?{0;|I_VQw#?q}H01lOjsiJ~*iW-1uQEY9d7Gg-n(wb0~m)iUFg zznLk%&0vAe<%W*Nv&`ZPDJIs0DY`JOAu(54)gGYq{4R+;osS3K|!JsuB7QPeI7mIIb zN=2~>ub1z}r+&lvFYc$1{|qx=I7R*a_R+LEpQkQB6&14pS3xE-S)(>)^Q1nVO*QV_ z*@UT|XHyk=d?shD?-!wka{b!PBD~Wjp+#szxy~Wexe2vRkD?=!vfBJ2l zv%3oEu4oIdyrAPlf^ek}Eva^aG!2Lp;Tj1-TzTUsm`1Y+xR?rL?pq%gx!udBoNrxk zes^EGv=x(vgfwU2M*Jkp=nZ5wW*-*l+_J;v4^P*B{@FdpblklGyMNgLKD*Qo3RLb3 zz&)@1DRs{heRAEGy6-7D3% z1-57S^+;iX&Rr_HZ=!o)HgyR&5=yg2#GYV`Ag-L+P@hW3F8waA11_6 z2h)$>#H1%mrhy_yC3e`$JsS%+^s^KDUyUzLsy-J4r>4U6FMYsfA6DZFD{t<&I7sb8xJV<`wsEpfa!1B18v`R zjtzz-c!W%n@6x7`E6QLDaghqQm+5gbylx`)osnUPcsjFjCg$7__` zDrDjJP-uiA$;e9M8esqIbi2L@G^Bn1aNFH*yzhTi?BwQp<8HFPaBC&`9I_b`h}bIHdX=ys|Ht#t3*VKs|;nuRYvQ?RpwyDRYpS}GvX@I*u+(5f}$94 z)v2`Nsxk@UI-Vu2CR!9%4NZ!x23L!#L_~_K3}wYtM(e~?=3vEDMnfMn;wsVD#8qd4 zq8M@2skGv%G6~{(I!jzlv?#6`niN+Jt`=8`h!j^D%8ILu)`_dk!HTPlhCXJ*Rid$p ztIh;PG2*IIX~k7#TD{)gb~k#v{bl!plGsou8zMAngorH&LO{jt7voZ^i=pGb76GY= z#ZYJt-?0Lw)}9m$()nJbaMQ-4U?H9R4g0jH^=|wyMDfE2?SDypSPz?S6UunwhY{+3 z(mTTz_D75eVniT`pTBp%-FbiLHpAfk;d(%7v3kt7x9X9psp>JI=xqI{7F=tic6>%3H^4elf8TEb${bMUj%2v1HEe+_bb(#o1QIDHs6g%g$H_E zlkY2TFWX~Cpm9~E_4pyAy=XrUT79^E=-$`eE3f~suuR{Ii9>2Ds4$k+!s8%X4+uw} zpZkl~+y89$7p|e)FZWIkCX#9(jWj>++?FI&mKvg%65 z5t*(%h~Tm02uP9`5e&`ZFLFiHv|m*~Na;J>kWvF&^>6(ck3zpbb!qQ&X}s#5WL#go z9=|rp2#;-N#1|UTz!Y24giXzAf{)wRM3NfX1c!d}ooe8zHNBO}_hiKoyUliw-R`e^ zn$#y^22cviNbR zEt_E%;-=q$x7BJW(E1Lk#^#ewAvDmg{6cj9ZN;VLfv5Oc1P;s45gFH*1G*XzD8f2N za}C<@Pd-m)vq~{Oqt(E4R;wYQ-t5d)Vryr&86N1nP2T6-2IZ6ee)G-!gl>4<#G}Wj zwMTxa_?W=$Y^DI=46^Yd0W3bcr-6fgws#H}0|RBg5Q%}!<;YrSpndW6?(nUj)#j1I z3S}G)g{VMwa`Sa}e-MI)@05xS#NE1tyY%z7GFyepfq~LLMkX_gaY%Ua!Tt2){;ONq zJ`8KVt#Um%i3y2%Rmg=)VL(Tg5YAnvOA>-+sjj;qr^vb^Q@saFCGZ%zgcz zw|n{Pkplzehud5C12$LZm+n`!#hSU`5;ggK4yPc`Ln1>`_kzamC%XSL>S_K2MrldL}*}|n9z`L{Mkf>#MUM*DmbL~^G_9i(0E_H0Upx7 zxxG~Ak&F)HZs>ZrBt~R04hh|zpWDlg`}Jn`8|%AUQGuSF1+`Mqqf#lQ3~r@<T6jsDl9C0l`iOTIr(e2+o znp%7Nuq>*_4_DO7Y9&Grtux36$a&&S#clCV^6( z_Ur#MOEHr`Dc>(qikSpT`C*At%p_3Ck4uzdCV^7^wnQmr5-8>GOO#?IDbFh|-fwT* z_1}NCZ#Rc8)ulXfLZqarO^uwymH*dDoA4q+T;X{b@1B@@PYmIma|81Xqi z)%dTWgfbhm0SzZPH&9SIML%(S=9=J9&|5(QEWFVzpem>w8XI=#^M?S`>M1Ig0#hsA$;E(?noC+S{#emJ|;=i-4OrP~MMwlQ#P z)l_VO_}PPhR;DG-jJp|Lrf)dYaL$px5no8~;r5T)-GAj}gVNcQ}j+m>*B=?Rw&D z#~MZFB%=fQpr~+yWPTd3I{kKebA6e9I-^#4J}FgW+$E$26Q{8nP+=0w!s7tI1JVSR z$Bd`1JTgsQc}(b?>{b#Bu1#EZcqqVMm+1(m7hx*A`(3D;PbY3S$>-Z{lN{67N`z}` z9ub?GhZI_x1ILDD!&5u6h;cKsxYWul0{X~bgK^MW8*9*z+&8fXK5bzIdemIi=|XPJ zNH0SsE!>1udbo)Zo4AUuba5lT*2WD;yN?^dS|c|g&`hOHu41`X7T}P+)XM^1YGwf* z((|(zA~KM^e7JRcH@lzu6(_H9WFUR9-|zPCcmF-Py*hR8=KU&9z7>d&#J#=b#fI4R zE5=2_;Q;^Pepiaw@f*H6F+Lkk?tKBfBby?`+l?kCcJsRyBMk?LK1k8q>N}kQ6gYg-2uCc^ZTu96z@_c zMBybkHlQ2X)xbEutC59KUd0i|dL;_wK!3f{Z8XZfC>KTE&~797&p$GIfepYeNqlw4nz+@uJcLkF7wruGP>SCHH?tHg3AozD>)re9@11PTqlSfIG=5kpMk9%mqG;4g4#Bb& zY(?`{;mbCzqG%bqil}VuDhg1`)SnexQ;%JM4$7N)_N?+2po8+JK0d3w`E(Wkxo}r? zr(ZU=pND6m71wrQz)c5cA**_lg{3%QZY#`rXu6~OhqXUjE}-O=dCe+$N#fog-9^-^t@{~9^{h+}9DGj? z96U}11V2+DE8azOfKMq<=t&A-$$u1Z^cn>aIzIUd&Cv1?B?*+0{6fi6_68*hl#=AC zWXVghl7POGvze7{p%`i2LNQXmlA`DXN)Evh6l_IrP~poip`vK{g^H-`94ZP>%hW>@ zT#TEj>7cx+uc-NQSua2b-b^6_ey*Dl*Q`RFsmlsmMU3Ess;M>fBCE3?;XWO3lnVpqdy;Zkd;wIWKuh z;??u*&Gyj$4w7n4W`Q}+v%s9YSzyODn7f0j%obf!i7WV{f+0Gg3X?of1sl7Y3Y>gQ z1p_U4$5P;H-lV30GQ10^IdbtDDWD8*0JF+4Qa~BU{q4?KWf&=-41bq$RvAW$P=?zK z7FOk-KkHr!6MyWi|MPhJS+{Ac`&|M2SvxC3Rz$yu=H{pNBCHIdh+#Ko>2}7u^0v%o zXr%S~GfT0RjhfBSNIUGU4y)$aTw;$7$(fzA+oa2P~ z#3}luz%0jy1k-J+FZ(Zz*{(W?45X87swuMMLFikB*i~Or_txjV`_aiXV+^64wXIG* z`2RgWY}~6&*B|}=b?5QTo!@oe{jv}KrpjN!>n5B_^Tx^E(!_l|-~R${-L*c+jz%og*Zj4OMB9{NM{X#E7fntA zjl4?=4}CMXq&BYUGz=8dW(LAwb2%xYkT#Q|Hmwp0Y5gTF(Z$m*SqX)-euJzjtq$r1 zn(lt^mO)}r*(9hYZvu`QrvM|NQ<$RKDdb4<6b5=DQxO54_N4(o?vm1icYe`O8e&qd zB)Zl)BwX$xf~eL_6tvn`6k6yoilf?V6k6^&f`;rrmMNQ1{Ca)(a_XL*&NiA!J=l*N z3LMOLI2?AZB4MpxNeb*#8Z_!r8dvO28a?Vu8V4Qe?;@lzQa(S^UbJ#RA>;Xlr4S>> z>iM(tje95XcK>O&|GIHMZbJ>=29%S60p+d$45(u*DvC~*tROj_APPfY6F#7sgWO}YUS zVj4GICLxg~yWx@twQjT&;{sv4IpCv4w-ln_q_8A1DRL|#1IE|2+J^ z-S5y=P8eBF>47n&dtv^zbFjnCz1($u=AG-evrKX4jgUa&xmW;}hWN2XS)$Ak{ZrcAZq+GMF6 z4+;GBL>rJ#+?jYt;Ai@L0wx|3`1=cd0wx|3_= zr4Xk|Da47g5fU}oh$%?25*-y;{X7Y>8Zy>nH89Ds8Umf*R{$!pH8D2Bf!=GY89$X` zGaTr>vYPQZJrNG{URN|d5f1cTRW!XE-Zynk)%nhVE~gkbs8Ng?Bq_!Xx)kFEakk<> zp{>A*OluI(YCQ-ATaPQLwjPbrt;azRCH>X_EfKddkSHm+4a)V#VjxixDW8bNK%yj9 zJ`sz7L`k@OA`-LeehI@}^td0Y#GME+nwMn}^pEqI)cB)Xm!mV8i)96O^HpS)6w1Z8^n)KS#@)WQPAc(=6BOg-1a z0>yYowb10n*jb<$@49n~v9mxi-kIkXV`qV4{Fbn}#n@S(m>>G*%;pwjW?B7w{rQU@ z1~1NTFK-^Mw*AWItIg$pCk%#Gs2dHfP&XV}p>901LfwF9g}M>Z0F48i0b3Z^lq_** zQ{&RuriP9OH#I>T-PBMJq#xdt9JTSyL<8mc0nWse4=OWgpd3Fq%`RsK4V2@Dr`hGq zpn-Dy05!WDJ5AGU{EyPqR~lP&H?e}DH?e}FH?e}HH?e}JH?e}L*RYhU*T70!FVRrG zUIuc;UdC#iz0AN_dl?J*l)b$~VlnqR4-_QZa-B$L?{yw1NH*p=QC`p-9wqNOh&EW|JRVGZ~=hm}+>p(6^JBse~ttj-!AwCN9Tu)mdDcz(Tik8fQ6 zN7`|m4^uo4HOyr!$&o|01u1isB*~zPqGW&#Su*gVFc}~;<3yS=10hb91WK7`i)CpF zlqAWu;T{UP+^UB*7KN%{Q8=&>sFIKfSX)H`K}kX)q^KVe)ew(}Ehf;ue9hQ^j;K!e+2plC{xAb~|; zkl2PiNQReL@E>sAs7V@6f}{If~YATo+p>k`|bUm+vJ*FeNWx>Wl<;_7KNh? zfhx(2fVBlC5R{}PLW-giQ4KkX*rJd`6m;xFLIRu+55)jQOk_h*GzCL3KoOG*ABrL` z!o;vTac>#^_J7-pSLYv3FMi$ZZ@UL@-`w7OyQpn>&sVKypq+kapq-9w=%{BKyr646 zQPj5unsjbKkM(ZhNV>P6q3gZ=^)xjfurfhW-Vdx)xg_T?K~Y}aOB6MWsiC^c<8Yi^ zNZ_VhNZ_VfG{A&cG|(!msPGU@QAILFQ3dlpQKevPq6&uImt3O4qBn^q5tJk;yUC=n zh$az~B1SA6bSGij=HLU*s~3P~>s{G?m@L zrl)%m#)dU_LxbQz-3x#oJ#P6JOVTuEmZa&+EJ@Rr2@=AU2^N(p3w(qpiwuknQ>Kg8n{IE)3yFU^ zef@^aQwv?4e%W6B;qFH_Z!ULlzVxqLc9Go+hx=bwbnnD`e(_H<$$J!Ds=3HYDvgjF$uoWC*HgEKc1(=dsEI>B)iUlZ>Q!GFR zg7Z_P*{n5}Scn)(_Cj8WdFl`g5r>lfZ&D1~ZQd1sjbi*^g!aEfF??G71&Z;95$bJE?tdqWjdz0#8z09J7El-}7-$>~ zxZu>ET>Q9KIaKOfEo1Dv zV^y?!(3azlQsKf2LXm+{CQN7&2@^e*2!lxiVWOc+{lrCv)x^OfA*q)J3!MssMM6@) zLc>Dyq&g%dJ??LuS(56IkkmgkYDua>uDs>ExjS6DJ<;6{1YZ5Jd;Tn4t2M?+E7bXs z73$2%3Uy9pg*w|ZK%<8lumwX?vP5@Np<~}u6C@K>5tSS@doNx zJh_L>pn-C{7tStc1`U+sU2=9gGiab3@1L{FvD4%i6c=x9vL8Y5MXlZM^*!r;Fe~4~ zn$0sgjB{ZPCc2phRIo7?9{r04B;(>SW5?o=$*On^^qKc47F;tY9UcnscA_Jg+K~#M zrvF@%KF%%}ID~8KIBDV#Zcd>@xT%B^kze?TRYj@sNeV*az)@ywcoG?l7)y-BC4sRB z=x#4B4q6kJ1`Wx*v^4Okurz2$KDyLt_S^Kx^=L@$M`2Cv`H}0~;I9fCnm%u&QBrI8nL^!q#DLz?+ z6d4XFTUKmCrULQC!k|JjTqbuvJuCd)B}4=gSQhsY?d9-MjGyiK3;XV)ZVtIR`QSHm zoF6ua`}KO*+R+``up+QtU%d7|I69eZ&ZCNhTt9!9$5j~pX0pa%aW+qC`q@-tRm>(# zvYAa4x-`?(OwO9{7NLc5vw5(+52xiWNE^y^9?)s?`Lk}DX8P^hAm>h~*-!d%(JS{` z*w43z?zL3ATRi^JB_Cg5D4n|F(W(Iv#lf(JA)QQ<;V(ytIWR;Li2v?m0HbDt@z ziI_H`AFv1zr1I@cpz`?}4T4mD%Bf7nf>xbkntk04aQ$2ji0xU8D70lUHnwBs)6|BQ zfN}d(!cyB+0^r1?sm|mJthHH@)&~Q%KYfkpysf0*{tR8b-WA4%k%x5&<-p;bHv{3^ zHzO-OP{lwmG@`I48iA$WXoO>rGy>s-re3KcT0J9xAXRDy0afmyHmL6yK}K{~x!i@!CC&w_dOPzubdrC+?et zkFpgL6E-BqRo>m#yXzbGINHUSbEAm`_QVD|w!z$;TT7AdhhdbkI8-Weg|Sk>5PMODNv5HK zjh#dVPWGaL1>t0#qrlgUNKFA{_*F7BM?Pj6DWD8La?dKmNC9OW_uFe{m0_fSGW^}& zS!Eb0pbY=h>9fi(QiL+PA=wX|KVRP;&Tj7>4&ChrzXI484x*LV2^uB5-5kEmDuE=( zOZaemefVbl0@T=YVzTONl9v*5*wyo0cfy44wtoZ2=t*50(2e>wpd0sXKsWN+fNt!! z0o~}Y2FCGUjV#OnR2)&bN>rK!D1qa7fD)c&0!mG1O)Q1`;U659S7z>5RZY z0;Tu?-N2HUVkUu7{GdLw6f+5w;s^GbrI<;e6hFAnEX7O$rToxO+-8BX5yeKi0?1%TipyWk~L&?J(h0-dN z2?z^x(_0cgjJOXm3iMmlPy3eG?l0VY>gM|LI-f~(fWqB}eg}pY=cY#{oSO-maBk9L z!nsul6V5F{XmN2qr3DtIP8y0hOVWXo(sj78-gW4tdL1t4CDXo!pymL24k*M=C-fxK zDT$r~3h|Q!JxN}Ol>-X#L;t)&tQ=5?Uoe37(*`}zBl@h2n zR!X?`SSg`vW2J-x`|(%*Xg_TNZmeWDppep7$&lnjl$8StDUEa5LaZE6NNJqY7GmXq zLQ3PDwh$`^6jB=Jw1rqXpb)>XK2J4P4k*MgtIsRM$^nJ^t-nbyuMjH-6!P~43bArP zA@1GVtH&$V9BXQ=W|C0K>F)NEo7ElM&5-`*M~Y7Gndfz?#YMu z+ut9q!>iwh1<~TtXS)WPNt7bPM2eE3Qp-~!C50-}u#%OTbmEoySQ*Pqpc_Vn5~WSr z94i#dh@7L<%AI2k#lCVoM6T~{w&Bts$awhewXx!3_iAyg?j<0*-ObtX7D*t!_@}#< zGX6Oy+>K&{M<+1i3-UKG#c$1oO+q)p$LcnbBuSg#q1&>7*}zj1ubC>8mERBVCi?Fh zH^JfaNRZw6?gCo3yW#Ulkl*7D5zQOhlb}GA?!<2Q@-6H_OzO8L)wr*k!Kr_mVc{6E zK54|)`ay(*H0d$BV0*=f^zg&%xj8dF6wr0)W-#sd8pN5a!fCh$6ocIID&~^+J z36$bzd#Z1LT8fzjO7Y9-GfT0OwA414m+F;HQM5HLQ7|HwICOGuu`$AK<0*-^DHLLF z6H&5nQ$VZx3&KXLx45SITYwJA6ZNbsZ$2HT&WmR+U!H#1+=lm8^aH(aL{0}rk+F)E z$aLD3*sLmU@#^&4W>PA&@A=DP@EC-N@p=5=4W#;LM5aod)am^o| zU+;$-FuEbTjjOA^jjK=1t2oIRszBZYiWHLEDeW)yCV zRwdy!kxtBQE{m+&j5>k0xj;?4)LW!;L@)C~!MtE)Vw(|&yi-f)@2vV|@#6HL=>asj zBzhWLnu62dG~?rf8Ug3WzP{oNfn)Q zOoXsGA}FacW#^dkl4cP>Nyq&j+sU*+U(y^R$?wwKMR{R@he6TE9w^bUo+R<;-KoXM z`c#{PcC0py(!1JZw2QTAKo*prwfKzACR0M0l*h@OxwR}q$t$`w8B)w@(m_&$8kI|g zdc8}GbCloW)_Xvk&EN)YLbETlX$=n1CIk^Gy`sfD+cjizC|xObnSF+nh~!gGIp00( zFSpO!>&vsZiL$pNXmH6ZXmIJOhz1`Qu8J6N-H{kH<318k-DzZG!&PLk=pZsh!!2Ym z$gAQE5)17D91)aMQ9Q>)sGB2#k}4N798+G>EFvhWaxtSXX%3O(cWKI_yeh)OplDn*z*#k)YKBD-QF#9snOf<#g=bF8qMDZ z7G=K4p1EKJ`2Q+|V;WtAkYHvnYB;LeKblr>w1%a3wFhRj05|V=A zL}*p3A`+6qTu+4NNp(m_3bQ{|QXLYK`WIcwYMGu>9TJlIms(nq>X4AszrfOxRELD5 z{$-Vxq&g%d^)IHhBvm1MkA3m=?jWtMy1`5e8%D847+7^9uHw-Z1ekL*23@%tRB+I4KaFErj>-&56YR_Nxo9i3*@AI3@efTBU%0>bu!u1i4 zh%LoK3f;wlW23R*spnY4xcyjM>O>X+-QqipgVy>}gNEe3?=Bo@!mGtg6 z`@(MC;yZIx@veDQ@lTZT$I5s&p;y)8ru3?KH>qdiqf}WyL8uHg%9IOEBIV-866H`y zpj$3kL11?RzeVl5UD`)Rl>u@(!7{Z!nRSc`?kemZVT zti?iNKP9&%R$>bt6OJhX^ z&=@Q#2x_z_U>n0lF}gKg6an&pQG~Tdj3NNURU0x2NPf(yf+2Nn(5Rww^Hzf)b}*YA~es>pIjBF%_2@45|IO}^pN>RWs#0f8c1)V1HvTd#cp<@d)*7;H!>I-K!Q zZ?<8rHf4~I)K3WH?{1n@g&Z}^c#j3!mLzV{pfaw~sxnY)+BQX{eI-z>ktJNoSX0kP z=vs42IOw?KschnD?^!vZkWvG;k7Qbik)tYy@m34HFi3`mL}Uh0EHZ^cHf=IdKoVS* zlElD@O5%~_B{4vMw?)#Xns)kpHYlzw9Bpy4*?K|tFXwQ-GSf|$hF_Lx>yHNva-#sC zE(%c*lZXZF5YZ?V0#H&00Yf1W0BEh$sUot~<|G(Wr#>Xn)3upyD#n4QJ=-3=^6jLP zEF`ql$1=3u(K43l?KVrj%OzZ+-zBKn`4T}x0}>QkvZ97I2dxW+2Fj@@gyA8yLeR8S z(;jw6{g5PWCCMa=reqeKy4rk<&Ljp(aT19_dlD6;LWu+fAnLKrW>6+W2qlUdWf)az zB?v*cC-?W;ufN}1>!xWWYNSjVRg|I(6SUVRD2h#jl+`7nT5^)GW&KDf=*FpF+Hgi8 z3|f4)&Rj!Y5MjsR~x5NJ`!IQf{Yg=ArsKzXj2I?k-(B*Bs?b_iA55TgojRU z$*E1G6Bfe-g|#Hckg;MTxJ22t$19BhNhU&yG7~LInaUu~HYqJsiH4V~#H1Ck#K+56 zVglW8f^JhmTD{yE)=+G}K63r%_G%b3zx{fB_;UL8t>4OwtAg0NENFGw zvvj;^yTMDh>0exVv;m!isDaTi)X0KAs5s(xszfFAD}iINE8$7oN?>TX><+Msp{85| z36$c6YG9e_Qctq_aPRglyx!eEKtpp7;t zN8(oS|eJw>o-}p9eLuy^ipdguVQU*5dNd^^?4Lcf#iPF3Bp2qQ~5$tz0 z4j)(cHI6@x;9+ND_IYh@V<{xibQ3^7`iwRS#zt#Iq#kPojoYgcnmVfy6e`bV64mHh zKM8=40{1$8czz=7bi|NA1N$6@koGReNl>7A_Hcc3w{F@MJq5+4#MgytH=sJi$E``{=~3bPUpS}y?YDo&zGXrK zZ7!!VYg1O$;8l9K9b*dnhuP3>P7z;-P-RV11v$1fS7@W+*DF-=-J{fiV(7RLML=pu zF*KawtT6>ltwAXmq#LFT?Qe9Nj)$#IuDo?uyW#cb>-N#(@5n&cVH~5Iv~q#RY@mB8 zkYjXD19E}xDL^jJJ^jZ8{*4X%I}sRlD&{Vzw26rt6{3bQR)s4^7|f>i#|JY^~x1gZR^c*;~PXmxtC-R%3> zrRbn+sPj-FG^(G7E$ClBMF$Y$k_U*PV;2ws$p^$xIOSf{0;c8#6b#aNVNg=%HZw_wtmUD7VxYHIH9)&{27i2;pqe!UO zBoQinED#EmU$I7?dRM$5REdd1{R%b)l~=MWF_FlxXiuXmE8C60K-sNu z=V7gt?q*!lwn#aavknPz+3ly@y8n+iYxhsKa=rvrR@N`y?zZRaLRsf0tCL@! zb|0^Oomhho7ZT#KfBs!aw+Je+ff())jAPq3398YnbaM^9pW@p-=~y=np<~_jgN}96 z4m#FNH<;KsWnh8}Qw0eYrwDRjvJW|S>>n~B8HgMkIxce%2~@KX4hISSkj(+7Lp_Is zgno$TfO$d<4ifqyS)WjYgM@zNQ=d?SgM@zlQ=d?SgM|LAW%`5~93=GbD$^&_;2@!Y zLzzCI0=MdJ*7*%+@@PQEy3v4+b)x|t>qY}Q){O>qth?xBV&llc1Q$jQ2^L2VIWUbJ za_o5IkP&I*kYk}^{o;xQs*M~R4ifs2g9GM9uE9Y!9hYl0_YQJaFEcC0Q!U)93=E3fIguH2MPb)FInglYH*P7$Z9nzAg0uq z2AGWoh$%I;0Vy>=OsO#rNT~s0N{w|uN(~TGYRm&tYJiwhV;_)G1H_aX1A&wpAg0t< z2&B{iF{Q>tAf*NfDW6!~M+L-`8WYKn@`(nBDK#bnDK$V$sWA~qsR3e2jfp@?4G>dm zOaxMDfS6KaB9Kx83@L}Z2)GFOu6!Gz5)`TYeFTiE+(@VdMXKRW!ZfUQE1?k?Nc$D@ zJg#{&p&6Ji>SWUKMV*XD7j<%MIJj&%Py*F1>Ns3TxO(KaJl||Cx6d}0f2>bloc!zY z-yc0XefE4EZX|p>IX~>XpQ&>HbRFel(w}lb+Lpg8X)5(}zug?R{e6UwuzKQeE%bL7 ze%f#DzW7RAla<1{yCm_L-n##J_VVQ~yFa$K?f~;+`2QtaJh;cJN9PB34*cDlRN$k> zr|$Wo!h7$@_?X~`1PorZV&h{1=%h!FK5`wD-4DXHQ8AQ@=)3c&ww(t+-U<_q%ph%) z`tw>_Yd|!M4MDu31H#cZ%Ke+R+5ylkb_Qvq)bF*W4uEElw7_UrDj{dKqhW4r(5i*x1S2^DuYIv&9I8ABU* zocifBby!U&ax2jxd3VfjPygX}9=48A4h)oT(z4&WpEBLucHhf|KdRHc`!&-yw=eel z-Tvin|8;XXgx`mJj%i1a!7V}-O6v;xd3U+FdGDqLrDMkAWcbP-V#oRG^{xM(DtZ2j zC=>4JJ6k??b3~gj-lWh13pXq@6!GST4wUA+I^1~vt3#)`unre`GMg7`2x@a=JqHxx zFYxsw`DLD!0}An%{qqVja%4I=|8{@ae(hdA^RTZQMp@HJEa+N^1#R=#g1&ihQ{x;> zLFX*6q;(cs*E`El(macWF86DHsmnPMS^F9uC}^x*4N=PjG(1qySi>5kyr4Nep`er7 z-R-xpyNCP!=9GIPDsFdx2^vl!RG?KqLApQLZ%=poZX@K4`xUjj-QLZcFJ7I0JS|Nt zo^|gH$Sr6#hqI#D9L|#Fa71gGLsGCPD_yiIJ3+E6JIC0%>`ckR>>NQC*%Zb~R@cWEPz% zwfPuVk{Bpkl1LQ3BvDbuB$0rw{3NTPt2Ud#nixVTQS=gqQDsgHA(SY(3B#C|xCG(q z*UkRcEhNpHF)hI29BKg;XHoN5qDRf6DVUUCI)yA{Sw7tGn%>`6XWSz(^X&u}B)pvSb?0vt%BvX|)+S z*OE9$+mdKhz9o^7#wF2!-U#Pv^O>xRp@cFCr(rmi=EYD#nS|FcoOzkcP_E9uJRGie z|Gj_VjBd5Z-M+tckKemD&bx>6Ys){eO?gEB z+e`nxe!l(v;j`Z&P-V}jLR|tS$~lVx<%~s#iUy&B75srGfbX6ATJm?@Y7&jjL4!@+ zph1N$%+{NEc+DBK3{ZquwU%P)3t9#!!fRSfkr!cNn5yBT-&jM*f`QIKt{rkl)yWP#tyW7wGIR1Wnx!c!wQ0C+K47Brv8EEGUHgxm_8@%8QdZOqJ z7HDz@3wrDi7LMc)7BncpdxV~*<`PyWD9Y>GN;P!~D-#suHEyNKi<-sMzh~MlB;4O` zK5u{7z1?sBv%NjMI`3an(%5o0xxL)I#CSH3^N>Y&oSQ7dbG#7G(}j4xUx??2g?N5k zh{ydX!XoF({bs@>R?{fIFWP3iU(peA0ZpTZ$73^@HfLg)YSjg&y}WW?8n-r8uEb|1kZsg)YSj zh5ASEmo0QDPAJqrxW8{$(S}7P=HC6zX45vTUJCaYCW~#U{%Zx)diAI=qc( z*-959#Zs3VNMLYUiWEy-YAAtH7b5MKp#QPGac}vrtU9}wAUr?2czv{nCnUO+L@mrs zr!6oydA7ig)v!3lR;DY=s1*!x0EEm}j1-}a@WQ6-nYeeA z*EN+wBF*r!rYxrRs-{v%qzNx-8UMlt;?>fAl*akG8yKI&E2g%o~}y8DcX942%` zV8N5;;P(-)ed4T|Si3PV?^7yR9$lB=}h(N$WJ;f#-5r4DJiiiCn>XzDY8O&G%3}&nb0aMn60Fo5afS0w>1XNVh#5DEO zL{^m4#6ahztB}}v4KSrH5f23v6eAJ{iWBirKtV+!L0*6X-|O+0|L)x2GrX0np+PZj z(481JXibbe_R9sWIM87$u%f*h1oT!9LXFkqO1i2?qn7G%&^<{%H9$)1hrFj!Dy$%kYT&XHIiD5&I3EKzP0b9kVjlH*tkn!^JHO(q(Y%%`Vp4i6NR z*7MUW#qtf!;emqE%Kp59X7KcO=q{q*eZq}Vh6x=86()2TLxfPUUm>>Qwha`RYcmeL zv>95m&L#kzvKb0Jl#H=~nTkpQgLKKRROu8jNSDk=m5zsv`oA&98(-<9f*8<56AWmg z4gplriGUIn)4*4>(?rx%)C9Hl)P&ZQ)dWEomc-IPb7~U_poEfuM27M_gKAuV3{XNz zRH_mz1ie0A{OR80e)!@F7jHB%p@SBg&_RJhDCkazt*EVm0*y7}P+85;lAf9XR8cb& zx+ZC-foUm5gh9Hb6A@QdhX{josij3+o{oo&`oE#K@%WSqVn7p32%t*K3n;Pq8~BP0 znuwYpnxM8cn$Vg^njmPXl3W^SPBih{UE8f=eAKK(HN!U}%Zdi%nFk5eXoqOkGGI%PmL% zLrS-QEABRU{_N%T&9?gy1nkCcvwilXcN$2RW>}iylaPQcMYoxj56CKD=27hiP z5kN#B0SWkswg5&kE59?nKmFZDg>A(`gqu+b5l>nXaiUcxDa@|Q!Es7e`#7~~En?iL zwYb!#wTN(rvL-Eu)>^a`4di}8)C!-@f?Cl)?pIJ+;W@b&y*mA}z5F9w0`$`;r>$b& z$&s;lvuntUDzhEa7gV;UT@_HpX0TEw`YYH_KHY7wE-Y?La8 zPETOE^mpnX9ixv9nv+HR2Nj6_b{=+yO`K8 z7ZPI8=Y-&z&oyA^bIriijlDTEW5<0dBT|RTv7ryLUX?(#uH|r$F!QU%cQ8$;!9l`o zE~QGSzzt)$lfvcc?(5gh?bXGLKkwYnZ*H&7A1=FXvBNKYb`f=vIFOAXD920?Ohi=(EY7v}}0L9^|yN*~|RN8DZD9yhVZ=up=lR;_zHFyh^Hk%Ae^Dn+zsI=K+ zP?~?$-9n|!CWF$3ciSyg+k7@GZnlT#>#;|3*$T5Nd?kb8%76hDXJG=cG5Zpb)U2z) z(U@&DQLR~46DZHFn()@FstJS^txbR$4AZHKnFLCywYRDiD@j2ceyhmY?cKxS^o#qM zcxH(S=2jYlqE<8nPf8a}Btb)TBm*Trf~z4y3f&STMIeikqG^efA^>I7^hGf7Vx`HT zw3^;%HW=Y5R{}H{lvcY+q1p1%=8_fs<<-N@;ri~z{S2YmVRRHlkI_*ST}DT7Trw&h zMRA>urf9ew%|ZDc#nx~<65sMXl0tSplBnf-Bn4===6nJZpr|Q;{^jOjuH&8 zI5sfA;)oys8{>e0q};#3(eV5xs^#=epzQBWc+1tBK+sdo%Nq>5gNr0kO3i3R7RIeb z5-6o+sv=8XikSpTsTruM6e~%=+`{ea;dk`>D+?8CR0@32>J<2*<8kS}G{O8RUgpufh7Y(ZrW%4kSFR*DR7=}bhg zUi^7^^KgH?yA=jLD$X@473T&rDK56M6j*5B79_T9J18}4J8s;f?da5)?KtQN-;OOn ztqEleB=W5%gQg89V?v_1AKPsXd{=w|(@(wPXAu~7WIJGWOLqQQ1jaqt!>9F4+2epf z5jOpfwk*e1s(zljs2VbEplV?1ooYy^H{0%7iLG@^D|~gj-);_mA-Q|1dAY6Q9^jKC?PN$P3BNdM9rXN&}#Ll-;6peTCX2HK3Vt6 zPU-6@K73_?xe?j|bHlX~7DqZIt}yH=7~&YH!lYhP!N#4Z0;fJy!3tfez}I?IO#x;2 zu~E&D4?;!?D8rAMv&yhhOr`PIBwra-Qm>3EiMNTWYqtqwq+2DZE4Kk@2)BU}b=$x; zWZOVNpZX=hNuO5Xv}gqeD54@bfr5~%zyL*59vWf>$M4E^jamT z>$L%C=(T|o_1eHT^x8l{msa#zh0}T!7@&xX+5`$huL1)UQBj*fkr!cNnCkU=lU`+1 zNv|@hq}L{@uGc1v(QB2UuGa>nq1OgV)N2FV&}#z)U0Ts=6;A6_V1ObjdKD;gsn{5x zh>F@2MVJ_-di}wqR~c2(tBfk?wTY_hwFzVNS|zCKwE=19wSf}#+Q2sS+CV{EDs?(DMAQ@Q}0F4MAZ780EJYmN3X7L*JtZjo5Pp> z6rY`~^R0o!PXjpqI>gD%_2ypj`w&w6B*E#qg1s3$p>{vkqTp^vPsqF7-wJjid_w>8 z_3!)5{+l9$jF=>Ozx}%V&-VHCUXg~uOcHhbL=-uM@QnWb_S21fr>}b|4o z5QXivJyAz_a(ne=fBo5cM8^Hjal=meEX@ptP{itS*mms3n-5(T-Dz9*|MA9c%P%}> z!2)86BNj1?r7IecMiCy9MhG4g4(0c4cXo)wabITf`^-+>B3z-c?d{bLYe5-mpSm z`^}x3Exs6?pk_=U3gbgJdPkn-vm4W>m3w&S@Xh}{<0(9~AqK{)4l%ND)zQSntB!V5 z|2wbUtf!kH-1n;@!vPl#Cro+7Sa-oi$GXK2I`&u*>mEj-dEMhEbgX+Ig^qQPq|mYM zCH{1*dtW~t+r65f57wNyBh9Hh(ww^^&B;3=&tB9T^KBI5Cc&cONJ-aFEcy-(8dmYy(nifS6Ka9FS52#FQHAfRq{_rqq}Rq|^X0rN%xWr3Q#8H3kAHH9$4VoHsPKuQe|Q))~EQfh#hQez^JQUipP$5!{z0Q*b+)0^$) z_HB1-wX?eRx(*e4R zd`g}{SZS8LuCmhHg2762I}lfz+p(pU<~9OXn%l832TD_*7O1t-EaM>IQC!25f%!B~ zgM)eTYSJat;2>ePV64li0%A&yX@J>ifS6Ka8<0{1#FQH2fRq{_rqoymq|^X0rN%rU zr3Q#8HTD51H9$dmOaxMDfS6KaB9Kx8#FQEnfs`5`rqq}Sq|^X0 zrN%@cr3MHovjt;W(RoT05K~&+=U4;8lo}I(qtpN~rN%@cr3Q#8H6{WnH9$|fQ2{Zf#zgX?e5wIrN{xv? zN(~TGYD@%DYJiwhVW(Ag0uq2&B{iF{Q>tAf*O~DK#bnDK$V$sWA~q zsR2UD@2&2m0%A&yiR4H5y#|OWH6{WnH9$5h$%HDk{{&{8X%_BmOsO#uNT~s0N{xv?N(~TGYD@%DYJiwhVW(Af)`! z>OLwUrqq~7ew06IfS6KaB9Kx8#FQEnfs`5`rqq}Sq|^X0rN%@cr3Q#8H6{WnH9$aAB-TZ-7Jk!bP$sy#WsC3m43q^aePjFI+rp(i`BAzHlL}NpFBd`ocxE zCcOa;=}UXsHTf-gEWqGgV0;EV7GUr$D8PWn0u1g21sL#HfWg0@00SNiFgO?#V8CMm z1`mS*40tTS;9^jK0gnY3d<+UO;IROMlR*InJQPscU$1dO3mywFI9dJ;ly>V&2Y>|_ zoD2#u;IROMlR*InJQiSZGAO`+#{vva1_c=KSb)LFpa26N3otkt6kxz(0R|_70t|R8 zz~E$1fB_E$l=k{dMv@m`!D9h4ov`%efOG&@fWgV&8!+Ip0E3f30R}u4U~n=hz<|dB z3{D0G81Ptt!O5Ti10D-7I2jaRz+(XhCxZeEcr3u+WKe(s4+WH-JCKYdFTjGw0%kg4 z=_v;30I&dqlfgG&z+(XhCxZeEcr3u+WKe(sj|CW<3<@ydu>gaUK>-Fl7GQ8PD8PWn z0t`+D1sL#HfWgV200SNhC_Muq8A)D%1&;;Hbi&dT719A<0R|_7Z@_@Z0t`+D1sL#H zfWgV200SNiFgO_$V8CMm1}B3840tTS;ABvM0gnY3oD2#u;IROMlR*InJQPrR9z!yc zyZ{Rx3z+GIrKdZj1Hb|dP6pqA0gnY3oD2#u;IROMlR*InJQiSZGAO`+#{vva1_c=K zSb)LFpa26N3otkt6kxz(0R|_70t|R4p!95rWF&b37CaU((+NvYnn(wL1sI$Rz5xRs z3otkt6kxz(0R|_70t|R8z~E$1fB}yM7@Q0W(BoI<*SDWT!b1Tig-ZmILr8ciprm(+ATPjzhXP9K*A!sD z_Ye1-Z};wzzU%+n9Ikh_f`N1cKOOJPLly6gLly5#Llys28UMX9{s(3Jk9>SIJobPK zX2(FI(Q(1ak?{kkTAKsE};U~Yv=Xu{II)wbNljoe}8zf-|zOsv{DBe)T9FqvZ6tt z85&g)4}}L6kqDF%2`cG|grdMm5cFYE425nGT%2PB20hEwbrC`WQun?@C zKl^!i*Rq<}DQ^mV0J~lpTodpzR&OoEWx!@#jE`F?Q z4wWR$#X}!^H8a4Pka;X5_8R8FxpuTzNbH4YORU90Vt zbZ%j-IOwimU3Ea+vLp;73U(ubN){zyAW^Uv36v*dF_0+Oi6#+?fkeSRG>J&e>eYpp!ip=oYoPz#rJaABn(dIb!aD=QJ_VbcCqnIzr&s zc7*WcJ3=5dwKtw7qUJmV2vT|LAyBy`&>%?VjYOG>20G8GGQ6U2|({k^+O%3eD2;cn>Au=le@p4|;2 z8uoq;$+Np5MZ@j}7BW1J8f1K7>>!YF1fgQnI6{SwM-vK^#uO?X6cDDg;_0i6FGNZY zg|vRaps~}jhK54gFm*K?tr7}p{XjyOoFAZfi?=jKP zqu%ypSk3iVBqa5w$3k-#&>4^YADz790l{YgDf>d4t%2YH6Quz^5nTiHMs&Ez1 zXz}T}qCp|m>T$MZ?%j)%ll5=^j=f#}FhY+q=!Y@1_+f-z{4~P8eR1;J**Z&Ij|e1@ z;r;gW_5ESHU$6gl@>^DZJs_#F9y1nXJu+#r9utbbI_&n3ybNo>)3eINhXg-u5AV01 zZnl@*69Q+qpLXZ#?lHhF**~^7cip#ie)9IkvyT|H&d>WSj%pzvCxv`?@l0KcjwKYc zdW>Jf#rWu_tY~YZ3b@3BXZNca?sFdvD#peqFiFUZU;2~L>m-;j`B&K@zLdVe*%a7+?_(*mzRBd;#}p0(-gm)ARZl z9zGL@i)9CFspjyRz*{c3yOgPyoIVxEi$y1NsqFNrz+Eo<{r2nbKU=qntzP{oL`YLS zgb_vQI7T5tmUu9aSC3q?{qMu}?P0%u{_*90yM4aB*}5xQH`!Ug>sA2V1nqe~_fkM& znj<+)c9Fj#!?VjY5|LeMk%({z?)rHx9z^$5WyhhyghZ=HuQq?K&(^Qp9oTO5umw%K(V^78JycQWqKiqFVZ~L7o7cc%i z>^J?I!2QW5w?zEn&%6EBy?tqTc}l5rBf60M!|naU-Q8|~*k1ke?QZ*Ww?FgmY!T9O zxOT!fZV4>CB!*Aez=eeVu0TiG5Bb>`R`?`>`uhPLXu8H1dMJJpLH;H{KoyqGa%lWG z!2O4H79(p*u#MnA?S|*G+sm7WE4LJSI{cS= zh2?#B6X2a2T)Te0!5zxlRKD{13f0cMJ~7n|k!vzmCXG;BMS^4P+;9H}vNbxEuZR2;2sMc?51Rw3#68EHE<^b{v?v;@w&! z8Vyg~5Z^s28KQv(4bwmn_u3{s|<|;OVY_4J>$mS|)f{96}85-`O;+oik539h48tHfgJBpZXD|!{rHuiGN*hIG zu+)>maA|?#fN5bU4Ve}m95gKq;2deF(^ItuPAeM}mxenlUHSUd$_B-yAFANe05h4P=+SgwCUrjgF3Su>j|TPyi(Zx)iXIL43l_aBGZa0V2rO9ivdmEQXo9d{ z(Tg(o(~uWepSP!*d-v=|w08!@%@_O;ni#FmM>^IRFmm9HumEb4VPVISeR24Z#e!HPG^SC?E}bJVANT z9YTGHW;GuxO&-?}q_*K8Q^7HlW_Qma$Vn>?=<|Y+NSRCt=xWaI!V2C50 z3X=vs6>L22slaLIQ^BC={OG5^Po;09fHM3TII9dJ1(e}OvRP#qDMA^;llHi$^xgO! zy#2de4ov4-jvb$C8IjJl92*WUJJ%AZcCI-bB=mmA0S6&v59@F^Na&r71Lg@e zI7sNH_4l12lA)%ig47qBr>6v~U;r(DAKQGS(rXNSRAKM|IGX9ri0#P@z4H2WUFiyT(A!!6? zMUB0`6`NeY6&1?Q{JtJub9xC8l8nal+dsN@`!wHn<1XVcC{W>O9)g6yd=j=g*?;yw zG~m~rL%PC(Ga2cw2+2sdxGYD;%SAb~aD~_g#lDonQg2F;e`2>MSK{7wD&?ci%S;t*;Z*CaotzH++rkBH^^W|`8e>wa(SBo6}R2Tle zF8l{w_>XjWLGI0CZEC)XEC|05TT*``ylw#vP{{`x;UF2mRFfoA#mSB!Kq2i|-~zVg z4FVL>j-@VO^R!ARr1h5{rY~6ug|z-6#FSPEg|z-s#FSPEg|z;H#FSPEg|z;%#FSPC zUHSR3`>mTdx9&}^UpKc`-7>v22pLdrEHa?nkYqr)k;#B^1C#;f#wi2J4Oa$~8?_85 zckADP`nw5r^wT2G>|kk)1byPGz~UQ)HEm&)&2Y`E1I@07du-*_+H8*-Rr(LH{E?Mw>7f5c$oimg9Uc!NwQ9DsZ~;Rlz{}-hmYO>73R`0cCjmm{o?60?P1{uvujo zDWD8L37b`hkpjx_ldxH37%89(KM9*vhLHlw@RP7vWf&=-3_l5*RfdrQ%J7r0S!Eb0 zpbS3=n^lI90?HV!WoOl4Q!d4P;W}c@vM`P9zPs92IZei+AX&KN*CpdokStu->yq&(NER;Nb;)=XBnubmx@0^Gl7&lg zT{0d8$-)Jzp_+7ePKzoC7F(Qpr4Vv*F)d)@FQzIx; zo()aa=;=V=FO>v9NHN-)e1CYpz1wy_j&|vm{l!hn)$l-%+m)x`!p7wWVD;?b`sQlb zLo<#mZ0Xd5AAQHN$JGOpzOZUQAmhA>icKv`g^!z;0;M*l!b3Y`4Nc%`txZEAt#>{e z3$EEe&CNo6?_9jzz2E+B=utH=bgKr4eXBv%I+*3aUglAwuI7Qo{^sGMPUnHpeBP7$ z4q_Ra-LE5oQZgOrSSpH8k&Lx*@$BWx(=YDD#$h)k>*Oh`nqsOK4Mj~k8V*@QEL%-Y zJie+coQ6@pRL)%AM)@BBX6;+#&ui*r6%mNS~fl7$vrV=4M*Bg+vc zUs;ZGY&6R;C+As?6SVHFXesiVH!aH#2jI%Vd7OuEs^eVO?e%SX-f~#zn%gR#S?suG%z%1GQhtz|i(eju4xyiJ~664h2ev zLWPH8%`T7$T+Ie(Xh;jIK(BXKZt(wfv%T!bb72jr8XTze6`*NwbN#0gzB)hbyNx`4 z&at_1mjj3M?V(=|8p8^UK{j+8)ot*?_^l_3BeeyZ8sCB*>%hX1WMM&vp2}3Br>Tj? z$^=FEQPN744=1yjCi0k}4FXyf27x71A+S^>A>!(K5TTM7M1r~!L@0EepIDANPT*iQ zK+!-s<+i1G>qM*|E+y8yI z-fypLvs)#4tWzbtE>s24Sg9)LM5-!uMXM@~iC9%==tEGef<}<5nhA;my{f5lbTm_-C7mCeFX(kvnUJJY zLBMvxsN^{mNz!vz#@e34l|(*=1^U>F)J|8E{ai*U(#v)(<#gUKld&p4Vg8X1m9-LB zilgq$pk)&vC`9cN917tIY@%vKyewIfg4L@?B#Tv~01>p6DsX9KO3^`iZH-cVyg(^B zD9;bGMyX7{y#?r?JU>%kti1VjRaX(GIE=gTBn^L~NJNLDSQH*7@`)}-Fvvbfkg!fi zP|03LkbofCZYQ!$6}t!_l-PDWLy3zJLWymgGL*OoA(Ys*Dnp4&5Vqa##ZCVb9QIyc z&DHy3({M>%NOEzONRnxtk)Y)ak|H8~lHyX?CB;a(CdCDsBD{_uHF+q_3k4I-M-%h2 zNLGOUi-3X&A2b!bBrg<9xT2}x1$oD&8`r&cI}%5Bt7NH}qmo9pMFnBa5LFaaD^#%v zCaB`7+MkLAy1Zz73OdsA)QnJM(d^WeqRnX;tMU^rwj0-bu>_W4I>XUWh}tE}PzYCG z6ICnXWyy*ZtX@STS*#)jh@h=hflDh>iVn(aYn0;S1xnFDdBtf0E0yWDw*VcKSDX~s z%bQPE^)~VPl5sblq~UKAiRf??i^AhXKGEd}2HEEb64vPmD%tA@5)ee&?L@YzVizHV z65EbvC~*-&D6wr*h7uPcgc93UWhik8!nXSnmwmJeYPcjXB)K?CB+0bSNYHWyNfD7g zNpY#{l42xXli~tR5ne}-nmm-|g@Os^qlx)hWN$7sFBDApAWfVXyd*CaOt_+{;01YC zueOKH%j=u%#rPp5-E!7UnrM$Rc?!-rgE5-oERN)dvuMUvIEyT~;4B*Gc5i?)_-fua zn-a?Owl|wIH;`p0>rxc1yz_Za^|%NQS%d_MMu7sIEI*Ne(4NRa3r}PsR3~ykFAI_r z$kckH_@F>R85E&HX%rt6=*NDmYNp@d5`0jgAM2MZZ~?x$QHfWMlvPnIvRP3y8oLq+ zWy2yU2+JZ^XwxFN2-_l9psNMr66vPGTZR#e6wJ#|L?%LYA_w%cAUT0dtv8Af3KYaf5h|2M@j-#* zL6lX^^c!4)4+<<_2iOZ-fUj;;;w*-;DvCumD~d*AS0bToSOf)OSp*AhS_BtiTLcSq zwP0K#-BftXFhY@nc^Qgah7pPs498I9B8aMi(3e^yj6%q3ZD?U)FvS^DjY)O0-cfeAds59f#QXNNn@ah4X!}(Lcya6g;Ckl zZ+J;wD0p;@v{=Cl^2+8W&iqui%d;`YOR@3ROVL@(S0v``m!MJ@Fu|s_V1iR&!UP*= zjjbKw1QbQXcd=i%_=(9lvSj%9joX- zKW!sc@MUP<*h;_Quu_( zr8WtXQQ;6G7wC+%2Z7Y=4HPdFOsbb6Hn;-C3kA2QNUR#C-|&*WP;h%fWG{F@UfFcT z^#GOa@@$OpQf$2SQgjyc6^VKKC8!hzOt7gfnBY{HFu?|zBW+luo!Z2*%uqCG#fGAn zWrm_j+cXrtEHe~MTBo7tMVU!kCN9(U?yMoDO`0TCdNs*vwrql5>D&~f*1#!Jn~zh} zT05smLADz2CD`YgJDmUyLF2@92E)9&w>CNf9D>G|=?wWH{3QwC5Hv1rI)uL>K|dP2 z-5kCw@aBpcG+@p%cq(>mWCQ~?bD-s#X-Xz*CPP~_(?I_xBQ^51taBD6l$p$O7H94) z%TU&(D6D6U=V8U;A~<9b5+oW03UspkL;^y4A_pxzk%>^9$N{}9NKPPA>y6@r0tIDI zgbJlmd{AJT7Fbj>{RWrdg96i_zf6G(@YRh7Qq5tEf|+bHx=G8j8LRtUWOukX6e=3YDO6;2t5DHEhnJPD;A71~O$lX|wXNoqXFFQTx)g=!Qfsh|MGzda z2niC60tGr*ej)*(J&}VJp2$R~PUL`I79=N-sr5$jL4ktUC_;tOC_X5#JczQYnSO&y z@Iis)>i~O!3-HyAN}R<|Rz`Ek*4U3>4EQ?^FO^e_nY>Qxlt`>|-q?-zF z8Ad2lFfT)q%P>Nbg5em7T!b-s#AuiGj5k;G*BA5XrmiuMuVo*0=Bba&;VKx)9J;Er z%pomU%p5vU$jBM&yrSvMrG~OcUNM)uF^if>ElXXPmK7%jwW3(ABmd@#re_dbykQVz z7Pk;+d8-hK6h0wxsZBy;R5*ml1v(?`K_E4I1H}sklg2<18(e|ng@Q*D3Zt^8-|&*W zQ1Iv)X|aMAFF~a+V1i9;!33wmgb6m#9BIQM z?bIfgWrm_jD>f9pEHe~M+NPoCWtpL9(mD-AFUm~XGO?{t@6H-h+N4QRrB{=zX3Hk{ zmCj8uY7LwswfQ(jt+jKC6lAOMUV?p|xzh>Y5HwCqXE4mWduyW;z#(XSna+?O!e5dA z4ngD6rbGBE5>!Wnc>Bh<$zpUxqNEeeTFIxdX$7-r&?+w3npJeHDXU0jJ66$we%eN? z;MH2Nni|S(o3ENX_pD{9WvL76Cd_;<9xuz)nC5A?c*7tHv$%yo%Ugv=r0@xmOKlP& zqrxFXF3=fi4+5##8z^2Vm{c!CY;XmN7Yc4qkytfOzu_f$q2Ts}$X@V*yt3(v>j5g; z<=GhHrPz4urRXf?D-!eeOHe5cm|#;|Fu|!XVS)`bN7}GRJGF^rnW1RXiVa0C%M3-6 zwrMDOS!O7jv`$0Oi!zh8OkAex-C09Qn>0zP^lFmTY}o|A(zz){t$|adHXo;`wRTRC zf^0S3OR&!~cRB$ag2svI42F4kZ*6n}I0TI^(;4zZ_)8MNA!uCMbO?V*f^Z0{$G>dv z53hGu+xOc~H`~j@_3n1Pe)r-)oYzh3|A+_ShFP?qG&)0c(8PV0`AG72iUOZE$XYrwc)#G%qQ{@ys zOKnqxjvJ*Ilv<ABmLrTM z-k-d@zQOvQx%x7jJ8Ogln$=P2;;Ee%K8wJyG#)*23iup3@r(zcv?zh2&Ad#WQ-xCHHL&VtBarTd=&=MNUI08hn7xpJ-U9T z>zL9BuSbW&ES~ooOg{Z>WJsEnHi@qbuO1p`Pp+=s?ss?F{o#6hAKhw#>rP)j1=7Mo zx~$12Il9AW^jteWCAfI;`6Be`1cMW~)ddrrQNPqs6Ty-Caj^Yg-MHY+Mro#94+%7H zcXtmro5OZp3@r>kC)wZ=My7Rc!3#2mB2V{QS7JW{oRsM&WqyQ&nV=4oj6;VTTZj&w z%tVI^Urc5z8iJa^=s7|mtE0k+8C~}^LL$w@Peq=B#@lg`u)iez$Mz-(#BqHLVu!}{ z7p6jN>Jut_+#VDtbp;h3jx}or0$1w+8VYH{T%)_zYkq^%v`Q$X4RdxwS|t?HhB>_< ztr8m2uKINj_-3}e z1A#Q%vwdq_8WIxH6qXCRVP^aeR@=gZmZ@6Q$90*?wjiPB3Wcp6m$B2Cx)2u$kII45 z>8TJG30oDW%b092vP|3`Q63)?T>Mld5I}ro-ps znhqJhA{gQ{6w_hHFKU)My7m=#mQc*9*t`=g*UNQSNauDW^}B-4-5OEZ5DFa5mAkpq z#Xqw(lKDiTB)p`f&qw{LP)8BUa94;FP$AL6SD9UDhQ#ihJl))FF0T*Y)~iST*;)5X zb<^>~U!6@(gbALd@0!7wS`7_p-I%E<+bt3PGn=az(INT&*W8zOw{0wG=f~*td$ie3 zJ`r5B(AE~oc0Bo_!$`2rdudWbQHgzj{WQ?%rFyNZZlC+koS7tIJx@IVYU?Hdj#sPA zL$_Ydp6_|TTQ3)e{`CMEQI}LnowW;+*e+9J(PO@!s?4gy%6$z~8ih7QC_u}l*zceq zpB@*x#okD0B?6Vp%($oSc`P6IFicTVu(}(n@^KHxrG&xeXt3!CBw(irLB}+P=@uEV z(}ZATGKSvxrXmMxuN}bFVmu&vNyJ!F2eo*O2Y0gLu-Le#)WwnMaW9XLjeB{dIz`cO zFGqnJ_i!vq3>^2OSX?Z7XI;aTyZDrra7`QcutZ8yeB6sNb7#y$pPROB%JMJWr$Ko+MGI*fx+j>x2vx7QBjco+rf9;pHYYp)&1 zkt^z;KLUH#PdTFEw8LS1jCxG%(;N^117ns%|`aBsYfsE#v4#!NDgH*S&{)T#8x zP9jxi{A95yal;sj<)Gz^ca=0;xett`cqz#7lu4*02F6q@W0w_~ag|J^WT~+gOTZ`% z!OKS&V+^ zl}-O~f0M4>uQuP1+jmX<1ZFL|zh$l8XN?@BS2v3}wBw0FF}$4X*_Z2X4F!*lP#D+1>#3V4?y9Yp)#y=VjxdKLYex zAc4~k!)pP9BsBa8NV`fNo0sjhixr3G)&fd(oB_8M$n)gZ0(k~?Dzj^W92IRXfMZkQ z!fSyj2Q6p3tEAz|eSR&FkiuIFUDvLb7NC;_811TPb#%^`3T3*2udiF4D?z6O5v^r!YoP*ZY^MQq}Bq)j~7_j5eR2xO2w6pZ_d8tplXmPwZWXFWZDI!xgoyZ%r=@jaG`g1mwD>G>G=#;!+ z(ngaI6Ue((8m|NdW^IBP>qLkWpb-DxrCu^UID7g9{wVM>cmk+hMrLpWrF7N^N;^v* zpO^i_ix$UcXK*DvPJx`k^F-MhJWrv{XMP6HaS>;59G#LEnZctZ#02uLmBuRph#5Q~ zhB<>{8I?%H3?8NI5`;K|=ed-y*bE+J;8cp>MJ$-XmAKLwJi?>S4Q_spkPQ8yh(ig% z&)`rDdj=2FC^^U(+$KrR;Kq;%0CK@G=rT zgXb7jkR&GP9L1121q@0=b_RDg=qucFu4i!ubyT)HlSjF0%;qd|CEcCTqhz_UFAvl% zoe?~W)T4t6FfvfPbY|SB%l;@4vpk+aOK@lUCd{99 z8X2fvre}TQvOh}tS|>rEC1ln*hJrImM9|tf?)bgx!@THm9`ssA9Z;N#xYo%NqSrck zDs|rRYn>b?bFG7;RdO?HohTVIQN3%cd6Xc+S|=f{aIJ%7R-y@OohWsec+9mK|U zt#zVIB^4@o(JQQVl-OEpod};gceM3Mgly<*Rk)Ne`dSByD_-k_>6AR;TE`|st#u5b zWUXUZkZTMN*R@Dt!=`bg=-o*sgi1~Wx^zhaW(_cF3b`njL;*D3K%p%yD+nQ*hPO7 z=`{)xs9joXlQ1;b@ewHPEPZ@lRUlrpI6k=c2nmlqTpJCMxoAUc#T2fLaZ(5 zbV{DJra+Mp6Ue((8m|OI)(}WBgtY^QQHhMK8Bmm6f*{rkG%h8~UL&9wIF%xJ5s7Pp zkT}j-0KucqO?+-gkPQ8yh(ifU&-`&Qq**^tqvRxJ{1}Nh+lLLYGkq8xnB~KUgc&|e z5Sra1mdKesf)32;5zF|D9s+n7nViij3@S*RNt!}2q)q{Y5`mq;)#{+|dj|U;F_cl6 zkr~`@R-D0cq)KXJ1~(+gaV`MRF2W185~EaSsJ;tcAjY#H(JEKR* za${c}s9icEcoeBe2NhsspmyoZxKWq=Q6gq}Jb{+r&h$|VRL91lwR7C@dsT*c(c?Vv znLlzsajM+xPbQ470mxM9yeHNIL{99QfJLk1x@!Y088cD6YpZ#bpwwCcFOIrqU@|Mw zskH-^x=TFl8iLHJgnDZUEK^B^3SM;1+9D#By~e=tsdJZEC*a72zE*`x2}7(slH%xV z5EPw~ms^V@$ii!qD1fv!iL%7jC{X})tr8`Qu36#;%GxE)5?jN>5t+421n5tVLR{fm2g|HP6V^IW>Mrq^ zYn?o&5}I4yrrC(ATPPDPi=r4is0s)(O)odBnAj zO@>RA%}=-DIA@F9 z!u$qd{`~y1H^d3X)Xc&dYj@nmV9Z2|u~yM!4Xt@tY%awjfzyL#v_C=wmhY5?1Wpgx zPH9Ns^oZ@0gdE*$+o$d3p?!YdY(aSlBsAJqq%+!9Bs1Dpq%y`IUfNghZV4n)p&6IXN0__w}pc^U$Vp=K%!Yo#&(5wbZ_Dg5?IWdI^%gQQZ}B6tiyglv)}B6tiyglv>0B6tiyglv{2 zB6tiyglw24B6tiy#GAhBmx$mo01>|yi{LQ;5x*6SU@?rQi-)i6hv((`%Wb<_?3RDG z-NO#qA&3JLQxFFx#vl$%%t0KO7=$=5F$r;CVie-Q#4N;tiD8HX6Vng}){R3f96e6- z97XgO7^DacdXgex%qT^Ixw8}rY=$Wk3=}9iO%d?1af(aDT zA*Z7v8B7Ww9DUV}xw|Ut%hd6#pyW-VQHo#gt0plAs}WFLKuUDk%^l{#ikx4 z5V(}-g+zkH>=%N1R>fs!g-aq!iMa8qT|c(#hacj0=={ELCzSwEwUq!- zRh0ly^^^cnl@teoH57-5RnI&c)=iAPQ!z1+V6DUeohpfefS9-HV;-BVj955uZ#c@xEkER!SNA{B*B>S$qQTV&kMzQbe8pXaV zY7~31J3+F4O|%z#6eRxc3oMHM?wc#5y{X_xK&X5ppr&@CjIF|rviGVs3T2gSls(AV zp?wCMS}xn8jS^xha!sx+m5MN;CLvnEXPzVa6A-NmSZXwIF`E~m?ZXt zi;4w~H^U1KU1I3B)NVL7H^mE+NVceAy6()4%nq-kS4 z92)slgAZ)5eo@1a=A-bZ8hn4|qV*I1iXma5Xbi_~`(^pOYqw1UpBYYAn68pAa$8Bl z)bx*pky36wQW>@OlUa6|3_!UEJSlu3&JpU7V#iQX`p*k1L&T%Sp6Tf1`@`zxdAV8J zD{5&L*taBDi;U(CPcCTF~j_=;*LZgP-=#`6I# zOX#pZMMy9!KS;P>a-bQk%phRwv>?FzoFIVhgdo5`0IF<2<2NmwESk~mZbL>UqpkieueAOXu~Kmwe~fCLPLUMvHmczgzAD1eMu z24pxe8IYzB)AeqQ)>D zBsy!0SOPiB=F=Hnz+m#bfW*!A0xFaH1tcH^WhG#<#p*&KA&@AmL?NSJGYScTM7euh z$Y@KfLI@-t_4giDlvsmsBnR+rWh$QbU#{VO$I%5V10^ag10_l;10`xH10}u$WT3=X zfi#pZ)HGP2GUEw!DW*V7?WLgHVoSlA>PkU@ER*FF50BMPDgzK93nrDqEU334yi6jBuU#*k+LVq&E| zPlIU?c@`ihR^szCwirGO5ECo)G%ow=W; zGs@pd6tpranW1y@FJo?!En}#T#=7?+iHM~U0;F7lJB-AekRRl?%#Y{oqTA%p+-Z~q zC>Qn}M`1wn&Z7)uWSUdRvbzasH2!efTixvz;-?v!kMh6zv-H$`cOm|Ef7Si2LIZvb zp+2+jSM{j_-K?$?-Ro-8nzXCU?B;20ev{a>nN{vsm7uPRSn2o~UckY~zWOuV_}%+w zcu@x&Jyorg8tp_LU_L!6+2Yt2W7e8!J*IIB+kk4+-y= z6%rh{moq#GZ@Xu}fqS{T!FErFtHiv&+T6B(=^Y4K?Vy3(WTm0wX^ZD^6PSYZQkepd zBsGP?OK%Do#2UynJd2<5B}9Jw-A_(W<$LlOUCbN?gT{0=E=@eNX>u?@j%SRAJB1?= z_!OFK3Q))r$v~k2X(9K^h7!yP1V2G4Q39Eu)6|h!hf-zqAIsg>{w0WvG^S%s?$WVl zJf&mJoJzvFg@Oc*R|hnWTM`IBuMQBfk>n>Zco|Q?g8TtFi6+V9HjxJi0$r3yWRIXF zJU|dA2L*ykcz_@<-4+Nc;PHx^sk|j+j>Zx#9yA(68Nm}q3kroIURo$Dxgta1ikBM- z3y9)ye&Xr;5>$oJkL;2;TxTbK4AZZ^3AAz=OFTbFhlI%wG+{hH2o#C@Ah2ZfgTR%@ z4+0Cwk8oqgM}VInbr^xja8D}{S%tCtA>Y5-_3p!SD;}19?#J)<+r{V4%ZK~!r=cl5 zsuq<<=UP-^P_IRGRE0`cnhLT4<))BT*PLP=Q+SH0-0D+IXi88q6-aSeH3~Un#i$A` zkSps)6<&KaQIQtNJ?fv`sV=u7Es!g3^s6qn9_{e^N9*PPzO-vm`YDxO{!|4WDk?h{ zDk?b_Dk?V@DsD~ZLdCu5TBLWAWIfynqJCku9NEP5}*&BTIg>yT!~}vGi3CxN#DKWcVT}SO;+lX50(4{&2!fG~$IH-uY(I;)m~~^~$#%Qh z607!Y%f+jZXW(J@GYDd-K%|0cK?XZ1LdFNvh0Nll4jB)k*h(WJQ8J}iTtJwWUM#Xm zs&Tlww7c5uW?kO3kDR3AqGif4(00Nx;Nf&564}W{hJsWhqXUUX<^X9%MuS+^Nk*he zq!^0{h|&qhq6(%LhpD@P2=1--yXfQfV)f%cE$1ewT5MhN)zfxZmF=i1o7l>)c%)); zzarb~T5P5yR=KF55?M@aN@fKrtAtiy^Ga$3POHRLU<1`{D7nS7(GpyV8HgTAawY0O ziLS&9M9V#}@{IS-bX{g3`bgdFR@f|j(RG=DX!(W5&p6X{nSp5erO3|^U6olqZE-(a ze*WBU#Vy<7-LT~8Lp`L*ClNzOIf5@qkID6tA5kSL3OMTs>CRn6(&wDiYv{kZvu zbnEnRI4roSy)3w?x-7V0JJTh7q#z2!Sk6TFYb=QUO_VCBZ&K`)n0b0a>8Rmz}Lb zDj-X)2CAH`GOAD=#Xn50AG+QT7GwrdxT`UU!u^d#lxh|+ zLZM5x7YY%`x@e=pGZI=3l@v&gHXkahU0AD;0;$n1QX;h$X{f&Gw@XlNMhMhuI#0N| z(Rsqfjm{IE;^{o$ah}c-p6f|GshUk9!j)_^WvXHmNRkDcz=NpQ1h!S zlPwxQLuA$>`ZyVknYPVf%rtNsBdX=mD2dGvPmXGX6gpf(q%cxiBZU{&Bq@v_!}IMD zPt9qZRCXXf-$JSMc8{vh4#ekMV2Swp>`J`+u_*7$Zt-I|zXyLbDru=n`yY=AnTCHv zDru=n<&R0lY$HD^l{HuN{o2mG+vGX*BU3)LXYCZGJbke4g-s0C#v8bcZhvz#Se2-QmjvaEFHhcZa;H zLV!Vc`0`*M-60+Z-QmjvaEFIMclh!E+~Hx+9lksOcX$|dhc6Gn9UcbV;mZSXhlfFT z`0@bU;bG7nzB~YTco=ktFAu;S9tPYU@u~^|2HoMygMD;Ico=ktFAu;S9tPdv%L8zS zhe3Dv@&Mf7VbC4EJOFoi7<7j(55OHB2HoMy18|3jL3jA_0NmkW&>g-!0C#v8aCgkB zDg+pGhc6HI(H-Mq&>g-!0C#v8bcZhwz#Se2-QmjvaEFIMclh!E+~Hx$-RK~_6FJc5 z%rsqM|5LoF_2Yii{HebwOYH?1W87Y-v8U{a8We6{)Yz*7M(&O(AZL#hVu9m)u0pUK zEL1FToX^_0V=5Ln&gXC3F%=6O=QBC(n2H6C^En-NOvM7n`I#Z^n2H6C^V37zF%=6O z=jVvHV=5Ln&QBC^$5bqEjM!d{dnSXSYia=k#fu7tuBjymxTb=kYibb!uBl+?np%c{ zYbqGJrWPXLnhJ)lsig?Grh=hsYB2(?sbJ`uT8@BgDj2$^79`-B3I?tb+q&^;Mg~LI z)PiImHDZ4l>V~eV1qryOf}v|_K?1I+VCb4!kbr9{7`mnwB;c9~hOVgv3Am<$p=)YE z0d=uJ;0DUs0(a5Cc|N;cxP8s(Xy81bqABNe zG;p3z(v)*L8aU6VY05br4V>o_HRYU+2F~+?N6I-J4V>pkkCbyd8aU4nA1UW_G;p3D zKT^)=Xy6>N*PZgtfJ686G6jkp9S+^o3l(rrheP-DQU%=8;m|$3SONERICM`hSHL|T z4&BoW7I068L-+KO1>Do&&^^6q0rzw`bWbl^z+#~kHQ&o=vhwkZR%RYLd;GPbL?&)O17MJr^BIpdf5W* z>2T53&93s)z3BWeXfV9S+^o%NB4?heP-DvIX4J z;m|$3YytOlICM`hTfjXX4&Bqs7I068L-+Ku1>Do&&^^6u0rzw`aF5tyck71jo&ksM z%c>q?o8DCq-P6k!IC?r9x~G>d;GPbL?&)O$t zR@qH?%QmqQZ&@k%sm*t7j^1#05Wq zQgF&)tjL&#l#LkMG#@jzH8yH&Og?UGYjv22$T2W#?6^qaG#x!I)V`cFBygIJgp^Yn zQaK&npE-|e$+8o8M_AfU6PoIDx?HvIe>|n%R247^hGM`j5rhHjVE6^>)PaNp&jTPr zE`m{xtp4FyniMu2Bw(beq9Qq&VSZY#}K=W@;o9)gT z!G_vxC8B4dX{>r_8>N&5?KI*3Xm7mTE`Bs0y#gDMh_lw@2PATwwH38EfE9o^fRcaL z?xu^U#lv#e_fUEub6obxtK=@Izyj@$@u==C!>H4 zv|QdT|I=npg71z1(yq3ff0BOY;^nHJ?4v>hl#B6dwRu?V(uXrYnT$me`kVij1kQxS|xa*A4<@APCTf+0Xujx`(q9;hS%^F$p9 zSf{H<0FbUB0f3U9JlO`<1p~Zv`ZMH()AoH}#krKQ(f;j1{5-N)i1>MufINP*X?$(TW;1(bK(pJXnW;A=(t*ZYlok; z3h;LsJ(=0F2O81;N=J-Yw1aM>aesmXYknYqJrHYtNHag8nIF^4U(w88)6Cz{%zves z|3))E*eBW(uj>Kv!X6N>>;eALf}d1D#AW37QZh)ryV8|~tgWfyD06pj1qCvNA7!oz zkF~Rcz{NIJ&`{u3?oFeyL+9)j6QRJZ+?{5*B|?E)`3*M9EfEUb%5Sb&Zi!IfR(@m6 za!Z5)xAOiRmRlkexRv+iu-p=%z^%L=hvk+C1#abiI4rkBC~!MGV#4xE#X`r#Y6A)t z5epp?>kV*B#6ri!iUS-IvCuKG<^ab;EObn)I>0dz3mp^d4scAwLdV3)0~`~v&@r+0 z0LL_Jzw|kMSwHm4w61atzg=W6-XcpND(pax*i_gsIa6UrJa5N5*ef3FH5mq`cfvV| zOz}ttFwLVeaH>bcLeo7O45xfF4CJ4DOLLbDNPKGQCs6q(4O37gDynAZf$IJ$;`{jo!_V*;PTQf;uw_cLKZmlGN zofQG?n4NV24eYEAXy7++X+Z-IU$*eB^YGOR53Ih3VWNv-Onphi^j#EX>I)pEZ*H80 zCjo4J^Bn6r> zPf`&PkQ8XfJV`}FK$5?{L=}HOmWqggB!8{RlvG3nB>C%3rlcYwAj#|*le&O^;fjcW zB(s}bD5;1DNHX7$6-p{10+OsR&k8k_F+ov9br%%yA|@!RsPY0)B}}8!<@)jDd;9RR zTl^&+d|Pb4w7I)f>_t!sT#TqpxE|J+_`tZvVA2B{3u{I;7Tg`$SR`g_W5Gc{%fXGo z<5%rMDj-W{Y!z8$RHKj6Pu*9F_p|o%;$^j)Zq}cdU!wPk1~zKT2t>A(5r~b4GJ>O0 z0sr}x68+t(;N6CR)64= zSUrMIV)Y3=iPbAuBwoK@u_Ss1na}GR3-Bm@#+ zKVdT3g}VwNkO*5r1&LJ%fkfC6DoCtC2qePRP(fl1Lfvo9#WQcKcJ}hLT6WKwch6OF zI!+0V)pJT{tgcf+WA&X98mse^(1(4AQzeBTVKwX4An!%Ur(rii~6Sh@0XS`FhDS=GbTG^bo%u19%CTy=& zWLBaCGGU9YBC`@DkSV|ZE2yP@3RR*6GUZo-Rb^J91Ty8{fJ8i4zK~)H#6AN}Nz=rQQLAR^o(0 zD|HVbv=S#2TB&~kp>;U*(&%)vdTh6x#W)dpV3uP3|r9DWf6a`-e*4!?{7IeZ!@hhIp6 z96k+{!!M;k4xa|f;TKaNhff3L@XINX!>0jqhTp%YE!q6Rm`|fui^F@JZrhc3CGX$u z&0_bJv(-{Ub70o4B{bIRxrFAZj+)~-G_UH=ysks@rVfp@rBiaHto@x58hf*+h^JB< z+uJ})J>F&qlP9tB!C>^ZL0BCAo(PL3wn4%o^LI;FG@wPmO%Dd2-!3Xq0-10SuOhP& zC6EaR^eQqdQ39E8aIYe>5+#rc2l^^9D^UWOaL}(Jvl1nc2^RoWWLBaCGU39Yip)xs zKqg!;RFPSU63B!Li7GNHQ39FHzE2gMwK$>BN?il2_9}5gp_Tdu5L$^73a!*RfY3^u zP-vyz0fbiKghDHI4&fg2 zpm|(TLXBaS`nSw9Ka|-?G9jwh}#1>O4R;9o&>i+kmsT7 z5aii#TLgI?&`|OV`Wzu=lK{sFgysh+94Y90;N1b7ej^Va2+a>tbfL94fzbRQMHgC& z69~-@QgorUIDydoAVn8iixUXVFYf6=YjFah`Q<%bXe~}4G{3;73$4Wogyxs{bfL94 zfl$O6pRTkTDU|wCno})OD798A0e!RL46Dub)kGEVRApJEG0UTQx5h(oC5CIG_0xp5j zEV*?LkqAhFrJu+I5)-U&h(tgVEb&ApTT%(p=x(!m>3&e|{^Z-!?gwF0kp~oYfd>>d zfeh63fQ$^(0uqO=14NXm0Yspi|3tXSeIgJctq9d!Y6cLFp5)__&j{J(HD5+;VRlzu%ensQ4 z1S%YlC|dD&bb$)UBLcbaR74JAwnFkrfmEm1@mUiUlurtzI>nIBYD=v~3Z%-#u~NvY z3w{7pBL!0B;<&QZTBN45PF|L)$MO1c);_H^Kg4gdty|92T0mp>umT#pgB8%&{i}e+ z?p{0^zjyI?Qk{#===UuSN1$tQXreufLl)>*92!sqoPI^;^NUp_N+8qeRW)Q*q69LX zmQq7zB}yREX(u&gR-yzlomNsqW*y3CZyLV;uF#%{Oo&v#2QA);@RInf=I%#7%zWKK|_GF_EOPVTQPesBPpp2%`Q=(#L@Z~&P; z$#P&Hz|Y_Z2axHV#Ru>+_`v~W`f2e2`~-f}x#zp~8|4ims;}RZfOZci0qsVM1^QhI zi%hj542R!uaHv32!2zS40|yVZ2OJQ@ztaOS4B0}DCjnBNYK~{ID;}E!NO9^WPYRc0 z^zrWG`>x%t7prcSaBu%d`#|}Pq9Y9pk5Sl>iSX0V(LjDky~F1 z=}f&TBm$Wx>rWA5tR7V$1yW^csle*js|ut*sw_PfSZ%4bNHay?kL_mtWwzKY2>TA{ zBPBCtD;}9KTly%B$QnqYBo{+0IkGC^>9FMy&q%A2cwTJ5#4~~%$5c)%HLrxy*nxOV zEv3=hm8w2F5RWN@eDT%UGxhfVa{2tcTz?sMq|ZLgXVofi0e7|r^SF_M8)QGG?$Vv`Dlpd%I3P^5lu2=%J&1CdwisY674cZ$Sf&>q?57MK z3mnTq0uQzwQ?bDDQU5F*&oL1@ny+{5YPEcB*UzoEd$=nDciq1eMwn6xQ635?bVe6& z1d^P|rt_YSH>H40;Z_GWktrB#3J}<`hA_EeWu=G?$di?(h|jJ8<#g6l>LyF?r(fHL ze<$P4v}d6b%?~xB6WzY_uZA9 zqt_J_>Ec*?Y=IJ4#HkBqA&V)Og+y-AETl7~vycc>C|N*@7-MC$0x6Ixt6v3Hzob?m z1yY9{qv@yq>q1u6`e(HoDUd41y9y`3KC9J8fmC_tMrEnhNP$%NQkhBtP>WQqh5OLd z(>cB4iW#6^P?YkB5ugf>m;kEqhykDqkLdqZctrcJ!XvtWDUWXcr9^?=UqY!{e;JAC z{AE0D<1b?~eZPzc)F;{YO9=h`RErY`l}*2{&{~{8sO-*lh1TK(LS=8RE3_7;5<1#{ z-#+iu_h0S0-HP95+pL@B_mlCs`Shnw_`IQ}Jcv+!a4EBfn)3cE!Fh8s{xokasW@vz z9%R04zbv1{jc85tKjTllhAY>J`pj9q{9Bv)z+51OtrKA3Rn$9_@)Y za;+8*?PMW#LB=QJZnLA`@EGllx7)>!=HvKow-sN=HsXJ}2oyHvPXl0YPkbV`NlbLs zXHJS9q%;)W`*w(i^s-5f)al}R_kPoSoGrKFcFE25$AyUEtM2=urjh^n{bV9e-Z*4o zh52Z&m`RDceOY&rBJb6{ZNF{)ZiTw0IT4rF&aAARB0Ot%-PTyYiw5|*1DnjOqsUv1 zG{J5A@Uk8D&h;M~5u}q&rM(}gKSe^s*iD6q37Y`X!y*9_m7r)?JI~5fJ6owK^_?`?s+Jp#-~Z5 zr9KCXSNt3@SNL=Ic>T{IK{U$g*p0?C1-2VtSb&&tRmafedjo>SB+AtWIr8jV^9vee zrw1BjPf8e&pH3K5DwA?}KZ){)K>p-G(bUO916h*?ff#iXCWp@E3XT9saMA?FkjM}m z0g&JfKsW|l0*e4haE2hd1Qr31;0!`^2`mC2!5N0=5?BO4f-?}&C9nv91ZOCsOW+W| zb}=mXz14qes_JxNac!dudDhG5*lhjNNA`F<)QILl*eLvQJN** zj&is}N6JC7O({o^=u0^$i0w#gN^_vQ6NU!Ji8L1s528n5Xn>qZi^1^NatdgGoJgag z$|;}$aw6@9DyM)3$cZ!^s+C)4*U%pTJmyh>@)!4~umJED|`C4FC(Zi!2QZoXYyl zaY{o1r?LujoYIiMsjRger!*vRDl00-DGdpn%DTyMN<%8A@_sUFBLCgJJFLGmEo7i6 z`uoQ$i|`F*Ap=bzZxwS)A~%%9tP~koVx(>$(?p(;DR5~rx3s0LO;JnRsROa@gh&Bm zg)EOzjz)W@%k^WExCKn!tfcP>6WfO31}5=6iSA3>(TS(W&0qSZtfN(-%b(2 z8;>lz%q>bG%TfQU5Vx)TY*@!?!@xc&$zIF?Bzv)9CD|Y0?e(;Qbr6_0Fi;&jIAfEN zID0oWaVV3bIC~XoR;prvv6RJQfnzyC;KBap?uWvtB><0fivbE5PFoZ-m%v$)a4I8Uk)%c-lj)7XM-m-@1aT-A8z2s|6sY7! zvH&rGGJvG9^ODb^#Hg84cgNUF=^JQ@VW!j;v6<30&=hK>1SZj$GHj*D2h=yMZneQgSvxN#OWG$9r>QCG`~u83_cu#)D0 z-l|XjVfpk3b3kv6B^dg=xr{%j5T_#%WvpT{XADCxXB`W|6b^=C+R_Qkv~e&*(-0?E z)5hS!Tdh2W@*|7{1TL-Eq`9O3N2iPB>i)Lfy=>R|6?M`0!yAkELmBJ%LslgVvNVII_jYQxijdau`tpXtvwb${IurR@rFmfX$VQRu8VFWInjFG5yEI={<;6g@6 z2H}TB8ZbKf{;+y^77xkGuVtg5fwB|PfwDWy+vz~c5e8x@1<~P@HZl>FwsV6jZEfNz zZ3nKq46PI(7F~G^a3ljP5AtIyk5P`~riP5N--F+gIiG1@zm*|B4DVt%pJ|}nz%ZXi z_AUYjilY0d`@htVMeuVIMNtYjfTD&bcA_RqueE-R*v29zX|Cv|=qJg#b6QGG#G-Vah=wy9$A;t=?bSGkVL+7(hi41_0v;gNQ{|4hv%}Z|y`| z9uSPXJj{u}JOBiV?31YLlZ?uoTL}iPt@y;DBaw;%yD@t|IX#_zU97*zmx#P?W5KvT zlr-Z74fW#~1C2#A2AhxV97`<9^SEfd=b@R%&l5y*AP)s08O(+p2Qn{kG(b);LvTFV zT*1*OIcfq`SrVHWt_jpcYFk97(x8!|lJlB0Nl&K{8x!hDDmSc? z#wLW5X6hKMK_X!v3(YJag2t=W50(KN8(U|Z@5C93XQxr&G5daBC z{Tnh1Byb2K`RVN7ShHxNGnjLZgGX}~2ajei4t^9%{wzFG2C{f`K}a%XN<#*lD-szv zu3ThDa)l!U2g#EvAxR!uF+orPS*fak;IeyeAr+A2_itqG`e&<<3doWVHHRx8@obe* zjb`mmyx@M;uEakd`|4+2ln^h543wyWG?XqrG+3bK;0bi8p+HQ9p`hHtLcyAPLP3ED zlw|}D_w#|r07S^xEfK+D=tqIbN;c5-5tc)PPh0AM?(#lu8VKSvNC$4t%mi!B(hbm@ zy$Q;kC5Qpj&*G;f7JO+(5(S)@8cuPBM%jU7`V3+TIH<{DLLrm|rGqI64~10*5Dc&k zCJ|~GWH9J57!VO=*uY!^KM)!CaFit5zI~Uy74+snacEek2x0CNrtL4LT*ZqJKq7tj` zzbH^=d5?sGoFcl7Bou)KWD^MoQ;7r(Xd3zAkCdG2VR9@k*z_Cdt95|doI4p2%`W6msJEmfR<0HMFPIpYj0>@_WiSC$) z9nGINtA%)tr)cT@zLa<%`pdR`?w%jZv~?y@R4qDE*9$r{P&sHMx`t3;rkYTZZe5|` zO@*N%K^)3jL!*dQ9S#c+BkK=`#x4kKmQkM_^YzoqZqR^g=nRxd5E@D+2MrcT3_O8O z3JSy|1O??L0|jdmfr0{YCzAjVk0k(=0f>-cOQrAwpU9vpb+^dt9|`Q{)1D*v3k5@B zV606S$XM&(#*qhtIuF#rj4?GqjInlu7lScDi?IfA0xEXm%*KK=?@%IvQ&3Y$PD#ko zbR`~%x)V3hFCWJ1#p=iNQa<4x4@e?H1SKA!0}~Gk1Sbxx0~Bj+f)tBz0~L!i!HPwI z*pUH?gT{iEf(FiI_)*|~MJ1p^!F4}h|4V#PeBLe|Tlpg}q-aLN!=a6Vr{Wxg#|3?k zDU|_vY%(44a71q8Ns@_@hl9Y4W=)QVkU}^rAS;?rI4)Fjp{Pa|%k{r6Usk*2lm9S; z^Hns~RAdm=^bl`)gf|u2YghrVP^LPw5ww9U&Y*P0X6;ONW=-8p%mPf-WlcdS%M8n4 zV%da(0e3QUP&hwJP%z+5Zi}ICwmSk0xI5}M(#Y-zFyKyZfsx%2V8EUHQ46v=JPf40 zsDEM$0`pyhtq6&7Y8(>flshEKseDM3Qv{JHryk}}!P1zAWvXPFAXqSSkZ|qHp;Bcu zhYeTM913KNUtH60Or6Cr01lg2wK$pQAF9AK}?0rs*SV6V#o_QD)sugoF#${b>^%pvy59AdA`A@<4~Vz100 z_R1V$ugoF#${b>^%n|m=9AU4_5%$U)VXw>)_R1V#ugnql${b;@%n|m=9AU4_G4{$F zW3S9H_R1V%ugo#_${b^_%rW-L9AmG{G4{$FW3S9B?3HX#1=An3~+S7aD>GGM~4hYSPXD<#BhYg z07u6RM_3GS^orpKivf;aGaO+tz|k9qBP<3u`jz1bivf;)V>rTMfTIIWX+VRZD{P{J zoeV4py27S9aD@dySJ-3+uCO5J3Y+f06&3_tVG|y>!h)bHY{~;ySP*oDO?u!82^vio z4`164&&&0f+g9ATy8OGnS?s=MZv>&E4#dqMbd;E^7^ou_>X?Ok#X`Miq291iVuEIz zLorS>P~9dD2S$z#Jtot<1%_rkfgYhL5HnCyQ0{n5!J1*4f&w*1_E$VSHi%OhfC$+{ zsT80J_5%r(0f><8l}cfY;4uIZvhkLP;4uIZvL%;@;4uIZvRRji;4uIZvW=IB;4uIZ zvcZ>#;4uIZa+y#fg2wU-cYhMbw0{!J?f)dO830HykW_L4AmC#o0GR^F zkTU?8!|uOK3Lr!F@FE#Z3Lryv@gf;a3Lr!F@gf;a3Lryv@*){b3Lr!F@*){b3Lryv z^CB5c3ME7SZpd)I{qMn_u5&-s(Eb-ghEKyk7vg@Xq4KvvtdAo<6cVyfB>Fk5)K7%; z1fGE@@Np8in#V0o>5khd(a!scL-JxpIYut+m4ng2dAoWN&rysHM#+0j8vl}!k>)%y z+?vxy2am>&{iCN1R&p2GpSqtHfIlx%NTY|ELY}~oQ^2T)o+1u&ZHs8!G%6x9AyY)7 za>Yu<0=`(j7E>yj-8ZAOK>zfojsYH6xgL%22{svvU-|hszT0iZ#QOu(S z?T_}x+wI~<^Rcg;-A&-*_2cz+`DM9o`afN59>wo=HjUrS=uqwUfQ}vJSy6|{icRiu z&?ZYbXnn|If^K>3(^td^h@Zl)#P?B8@F1Fln=^E@axQ*)V!C)*h!=zXXb3Em@%=kX zL?MeM=5G0)woC|@L5tXK{=qAlcfHy^6FjtXK3=Uh590MM6an22pN%sW2E_Bvb)pVs z)-G5FBPO5R(OAX$Cwox`Gup@Zmhrf;0)m0GDk|NrJHs&iTQ!;PU!5^)ULi|}O({$w zAW81lFqwun*&kaGk&-mJKf?xAw*(5TJXvtv;QRxYYwikAf&S^Y!1G zf1|`BEFmdM$50F;ISLVWab ztf+@wO-R{QQ8+~eqCy*KD!I!Esr;?80F^eP&lTuF5khULNIg4mvvbI)yrG$q z^0w|slt-8eC~vC{&Kh`fpxAta!zqWLkBHs9<>IeZd$;R4d6VopFmF7 z{hM`rasB?`vzR-?inBXl2QxGt_f79as34uS&kyniw?u4UgR|2~V9{NZjDulKa``M? zwAFrC|Fc{_UM`;Ys3-iFJs9*P-zsvP93XI{Qi4k39=h1vTz2hfvEl_B4KHq`a4tI< zAB$QvK|XX*RI&I)f}=r<#AhNI$r24`Bt9S0C?aHF6S#mdxeDb3oG+}D3kZ|9Bh(RA z$_0eU8x-mYE9C;hHLM+|qjew^(v4#dj#3I`SkzLHlJLiH;R@ zBs^cz5k#?qj)F$ZISQRA<|vM6DMz6}EX&)@oLENC_|>wE35W_7UY4jbCLk(UbXlUx zn1HBY!DWdmV*;Xr#g-+ij0uPe7Fw35A||(B--!Ljwfq4ha=FH1{bG&BihG)!@_@%4 z@vyNLfpVO05Lud7dq4oA%>jYUv<3u2v@syCAX|dPJWGOX3P>IxC|JmmM8hVU^(_@O zpyh!9LBS%<5LChg1O*E?Lr@715ELxl3_&G4Kv1x7GX$0J073FLWL_}ZL0!V51j%i8 zrw8dB^}$=(NK^IO?hcjSQCCz+BTXf@-2+^DGhOlJEfr~Z!U1Dy+dXXv%YE85RNm7F zSkBY7>foF$#}p`4uJbtM5Nx~uvDmJ~cXrR29ru8lqO&*KZL{P)d(d#C{yN^acV9RE z$lrrfe+>Qri24wT5Mvb{A!-gOo*@qv6`FzL)-VNH&(;BL0CNKD?a;2?Zu1~GVIxx}UdvSe1Vx%~WM zQz=<;kXNY|dfuWNk^`S>f_{8A2I28O@VO?C^E$MM4D}HsMP*A(jq-}dvoUFqq;kVO zX>7teX{L_B8g3Hyu`ta-lp8Rt|M^E|Oa}&v!XD72L2@*AO!}7i z(DVp%KreT=(DVp%MK3#HSBZWP-eU?r(V)KjbsvTM9{5CqWXDsVMS7k0v4VFs{?9+W z-Zg|=K_ZtJVxYcy62d5RlXP96Z#MHHCSgAJtBUyf&gow}zJUhhw&)gIpw5 zautrRYUHL=gZQn+?^gPsf!%?hF{XAdlz((%legCt6u(vlAFF(GpKDiq(F$bbBP(k%?f&WHA+ zzA#1aiPIl+rvDyvrayI@>A#1a*>4_^URCH? zp9CFRpF1wCPr^>8cfkWxe-64+A3DC&pTmy2F+AwXd=PYIK6c!h55f*_+i#n{+u3p( z^y+*XbnVzU{+&;uh|z6kiO~2tS%@1Im*2$u$(sGk@h{@@`C3dB>xb_39Cu>TwQ6_b zA=i&xfLt!t;=3GSDQvqY0RDC&z@z&&-QUi(o0q4ic|B=5zWymVB~|^iIrwaIePW&{-Rxf*?rZP(~D+uIlj8Ox_Bp!Y&svk+P31s&%fKp z4l+F(y}CTPyubW#_x^rza({Yzax%N`^-o@wtKEG4UOb1rZYHOv@?V;L`v7Lw&Gp&* z?X^%6IfD0_%}OXg?bsg0KeevOv^hJQ+}ymIp2H&$d$QeYUV5#2)3+M%=IY(#?fm@g zx)G=7V)?x5Rd#v#{>M|>JxKpNB-g?36Ojh9{xrXliQ8WjdFp0o&B^8Y<@vkww^Yr) zozE{P^Q*Tv&HQ=XrIsa91kjhYxLn<0U3}6Pj9L5i+{i<}6HoAoJ4yCMZpMR6n(8=H z|JjAu^ZoV9PW;&L<;$-9K5%w1Cn!J4zcsy;cs_ZveQY14wPUEGP{yp}mn_2xdL#dK zy4iw1cfmTlXfCcN=NI$S8TGtfG}n{qE)DqV3*C!^yK>H z^y+Q%?f!HroU?r}CT8O< zV!=nRF2*<4R~Ki^=fz5_y7i%mq!f4jbw9tMkEi$ApCa)xvhGKxhY3s@^kwI&IlmrX zh*kSpGddRkd$9ZVH0rkjM+YIWsyfrFW_CWkxwt$(!2^vX70*^OPS2Xj>|{2XkIy5F z5+v1$fA5n1bn)-@?uU3sz_-)Ia`htai0U4XZLa_NSNkxG2m2q@hDEnYVXl6+>nb+V zECsWhW^!`+_T=K~tZ|D~e=fR27#++OyG18LU$4_iGrv5UUc5aY`^tMaqF#%D`0=0i z{_^hQbaY@tFW$bp8Q)CBLdZS;acF-c`;Tu{FWp<_)M3aEE^hT7x{Ew{+sv=tU0$7c zb>DMxaVsu*cVNe#AOA8DC5u=8euwUW|LQh*pqHV#xqp57;r!+9>td^V;gEUL53@0} z67^B+aEhM16tx@p5fhiVmYeO5zR)`#J&QN{{Uz$)ql3A)(0}O9z;!b_o1dMG&!+wv z5HtJepi7-q`?y+})6fGi8&Rv~SLbJepzMLWe>MX3?7(iC)9bh6`I#t?-Wl&<0`re} zamv%fR}XnHn_r$yZvqKvBirw-LH3vZWV_vn;;71Z7p$VPcz<|+e>oLv-Dx-Mtl;0z zwu`5)`cK`txNI(GQPkVUq)Zg!=QnSHreT!D)d{YYMx z=*QLt`SsoGVt)Dd@@=E6W$96`dig_yU-!yj70%s(TsNoZ@6Jx*ib}tZ4s} zrZfDoTZ*(k8Xe7_PhVE6Bv#C(le5Y6Y{culMF&z11|FRH z&{I9szTq51&HRIln!5tHY2-J)BWs_+P3{FqhjaMAYG@uD&I0be&47w6}1uP4I+ z$4GPl9N&oIe=(k(^lvmwoCtlcz-4#&l@<0xMC}tNB8q`0;@$Pt`0DIzNQ9Ws&=EVa zoC$%5b@mKkaiKGT#ad@XtQ61RUQNXWaUpX-pAnBt2gj?~Toi}5r!qL65$8%>8`#xi zu9P+PTt#t+a}~wHb9HuketLazd6qs`x5I8iUu(Og>;{5-=?Gb+&H>8aId&ArI_7SY zo6f{8$@Rn>bBIJo$MN~)TN zm*EqkNJ2;K<s|`itM$DP3CUv@T^^nm$h~g4-Z9uN$39gdGoS;aJR0y69Nt2x8HZ&Oa}kje?O=H z)E)aoOtaUs%eUv#Q~wBh^U=WXFyY+4afI#rX6(iX6_mwSRH*!IrIswQG`>8!es?iDNq>_z z-K=*D@mq`C2A15A7pHvu_TAOl^*i`2)VKTow+57%Sfsz3zq^{8pS)|{cl!h4TdMAB zr#tmMM*scWmG}OOOnoGLeOZVvg1QYRaL-Np8`{*(YH4Kj4!Ld%>;9eC!wT!s zUk{~RYon}}``h<&(=q6qYVsCz^ z^@7uyu<%Hi$-=!}EJqKj3WtWPMvwgI9j)w9xje;soa5tOEUnAoE7509!zH?Go?<=D M@fGWG_^A5+54-GdegFUf literal 897329 zcmcG12Yggj_Wyft5{gMCMTCSsr>bR_wa2y<&IQ-gjNs|M%Sc%Dpp_82x?z@ZmDayz@QhwsX%t=iK`g<>#MO zC89e2drD&F|86P1@|A7T>(77j_TBfI`#||kCCaYFLlye&(?6FeWeWc@wXG-F+PfmD z|EM(L!(`R?N&kPBGG%IOd&k5jOU*Ct#iy36=Nw9zgFUyj!+h?8?^I>HX8y=Nn9sF$ znO{tDDN5nLWkB<}W+}GaGHnkU@TT?yBMGaG1yStSs{&Yj_tm(>>wp3b~;%i&cT31&)wHty@@uj#p zM6qh-^yWFqj$}(Z**q_m>HsGf!Ri0G0oXjhtG#!1-PF|LBb6zKDh((WC{q@;D^sHL zlqvD0__w$o{8lUFC)r-Ty5#eHmaDcKbJI`G~Kqmc}1Ev z>5A6o_O8~BJ|J`@SMukt9o+MsEgc=HR!*|IC|i_tZ%=#I(sXkR|G*#na=7qVFKa^H zBhCCj>XYWhOO`Y*>Fa9kr5c!@IKbNC-we>2Zb_$;or^nGbExxwJ5+L2vbC?bWpPK6 zL!I~Ap)im+#&f?NV+mTVrF9v{`RupjEbU44b#s)$lHc6U9qo&IT6*+G9{byI(#hUl z8bMs3nZF&VH<|9`Ac@}&(ns@}qqGke1p~PqO01Q-V96$CZ+mC5nF_r+g@eJ@H7-*` zuZxiVO@CgCC0Giqn*20)LN_@jfX=H zZ)bP!>ZV4%Kr}UWrFxqhX-!!P9u`m)0Zq|?E--;+`r^TmCm4@9h;tf65dT6=)3_-3 zg(=E#LRh0CTxTMnW*(m}5cY+FXepX&)JjIpnr3p`nWlgipj5Z(EkM5`6!rw-0hiMu zkazZ_QL)89Z^?r-l7u^67w$$A%k6ZyyzX!;77O%?)s^aMngz2)XhciK8p6%!xc8X2 zu8<=fb$Vi9Ph{XqFUIODw4u}s_gQdWhu0Sm1YPcMzhY(~jUd5J&;@(iLW((pF}Ew` z_6G(Q%utMiaAOH_tq%Ey35lr?OL%?GNHih}*PdoQ(KOK$)rXEJj*%?(5`D*qWxmPr zgT*24a78_?NW|lF9MaO!hdvfMp5~hPBlxD4G&S~hrTe+h@ldHJ; z*+qP)KMlF0Bej%2e#u}S=Q_Ntopw8^p4A=g>0V1EKRbAY*0e=7d@yABZAZcMGZbb) zrP0z6M>rVugnd4j!#&rjn~1{TDxwbBeFWEwUV*xdsQ1#T zD=lh|BjyV`y-u%RpbmC{IDw6W0I(|oOws|SnE;JZZ`dDpy4|F6!%YonOo3-R?--5rdf_O=i+ZeC@-0Hl|EdBPNZmB5Fyv-|NCHH9=ioN8FQeyMqyNEU`xM zVVxFNKuPBwX(D*zj(FG|3wqrNhbNUxcQwtVm5TPXP?24$d-bIqb^=#4O^v!pF%qd+ z7b#@|LL^VjnQ(a%VtdU=E=l$zyP$Z47MDcou@Hieus0rZc|(3jgzqHCDCk$+Rgxfw zGeIg=RtoB_KTXC}Q%8H(a*RCFR9-O{()`MWc5sGPxyY=?w0I)u3Ps##kVr(Rb^*z| zg3EjHZ)Ym(Kn>oPo4g~ASTy2vyW%jh%$=$rgQU@}H_2vrJziHNoS-QdVeKQ(3?sQ$ z>K%TgNdr~j^SXjDf6$qK)i#h|M(GQ;K7w{9z*Rcnb`vn}NW>G7aMVo-j!O~GR&RD9 zU9FSeY?8Wz4qwz02*d)R0cFwgNFrU9y$SGW9q?8YFyV-X;(<`WM~X|AB}Y@2673qD z_I8uj9dZP`Ay?E36_VGu%mRcox-k0?;4wPj9VTGJ5po3r;a~!`C1`1n-7ZZisaNkZ z>D)d?%o7ed6HeL>LG~m;j@1Qu+5(I?d~RbPT$dvpiuj!2NW|f8>0%nOE!mxT`NM36CS%(g8s`X;0Q4nhrLRz#h@T z4l}_#F^At7^aW!94-8oDalmTS(g}HJU4j;EYV1hq8$8-!`TCj~TbJpm2NLR|I_eP? zYRD0fdSm{G6Kj9Yp0Q~r3}Xy6eMj1|1U7&iHGhDd)*O#0{cGlK#QGf z!Lat^>pa-C^z`KG<`pKWD+*K36-PTln+*)gw}6(IW2g<^(cAE76VdC#;v5Jk+|XwO zAj0pm6vkj<8#f4aDZPaGF2k%iqiPWAZ2rZPyBLj0b+##6W7Oqw$332e!%t2~VM<9> z<%Jf3H{lHgBT*l;VT^{WReg!#Vv~Y))M0NT=1QRYc@-hU(RCnGloqD6o4(Y){MDBfdXfW>fM`&BF&lR$pB!x^n zUDmbYZ~JMzyG?A&!)PKHc1L4QU1U+o4`B2O^RVK+!3;*j=wqH~%j)|^q-FbK#7=r! zu%m`Uw56{%)!nPRI=>d59N4r(=m)!Ai^;u%A-qNn-hOeFk1 z@=+O*(+VV6E97x*Y7oQ`MwzCZX~M-JjoTTI$AQ+6a0ko{mq036s9R0agd-lu)(GYk zC*`e2n>B$}GAPe7X`>Fe(-{gnVb3wPxjutps{RbU`sbO1=JPcm?g=<#xUT{h zLqK-}GjCK6Hhw+HqxuZvV=;lv)WI$?!C;mLLw={v6N3u@%0#L^S~Tx+M=n)e%Shg? zOHEja=JI*G-f+M#%4kR^pe`d%7*h>8qO`_eVIt1($9yrY$k>P0w{|o&V8bM|u{1@l zHi@PMqRvRz?e#jAwu-)NVbi85kC~5qpuZECQLb=|HHI;?%n^~MK_&>#clx^EGpU7V z=`H-a35qSW*X#4a^M`(x5LswhsDw1lcA5YfAW>NR@E$TRbUqAODrS=`(WR1%{`V}< zgd-G<#QnY~^Yi{Ps8rScsH(H|s(xsK`h57##FOW+_HV!4*^+$~^7} z!#RX4vgK8Q^4i9XYaSTbT3^B=P2L0XyoG!FWJv7w=zec}CNQp@V*LW=E@keNa zGMXkQ0Z9X+R)8 z=V|W|DlBon#H4n+9FbTw7K!l&YD{s*i>Se}jWS(p2%BzAxm*Mc2E#tT%jef2X>jn& z2}v>Pn|&6`$On0)1?jzkPu5WB_A?-E0ci7)$s9 z5pOUKQyJ+3LW@g#l7~$K^jFmDjE4Lk>TE`2I<*CAslOhvr~?k4(;0Ajpg%?G{I2Oy zG|ErVmGGk$2>ck4h&%435hF>E<(ijb0y`}FfFt4Z2VD^_=^qJTS*~AR65=rn#D&Ny z6lVgxs)OXUEG66Yaf{sR2zmWZEKg({F&RwqtXfEm=94Bu0;(IP92wL)K&-Db+lwwu zx@SyUSaU9C-05>g(TT8$7=M#oHZ!qLLb5CGwzi%%GtQkd%-lL}F0sF6g> zA3*e47!>-*Y|Cz0DY~qVF&0Ko*bz>+ToI%)^p9d%@p_}qrZzZHZ-W98t3H(QhQd+o zxtkh9mz|9sU#CAl#C*KLVQ(6P)eANF)^U!z$5j1D5mvmmJT0;5iWbkBu&$ucYH zZ%_IQaX8Tz7aB{W);&dU z-P_FPXZYb94>`%0tVb5gtYwG~2~>i4Ua>=LJJ!UsQEN{%x2F)HPquXOCVsTP$LMMC z!AHagf1xkBsquQu=uXdAV_;%SN{Jp3Z-jhFx>p3g12F1q6-^6iOS;CS_l6yQn28~$ zpP~cgZpD5K^sOzu$u4++$=}U4r%O^uh|zm1M2-25-k9r5Fi#Yr0k_8+kB~dl%J|`) zb0i#46QCrdl^B~k+5v?2o{n~sh30p7;!!t@{g{N-x(vQ)q!Q66VyQ8$_^+|8kFq>< zL`nOcA|gft{%9z{QceafpM-Xfi3TrbFyIdRV^EG_ZS%{=E3M1tr5bY|rly>4B9glG z$NjNHfKp=mm2Yv9Vglsb%~#zUOgL=20s$Wkdc-k;(Dh5H$=Z^w9n7~+JxivE;kYXib&C?TrurzV%PcL*MPg%H%(Ifb z#xI#*ZoeZOc7tk^9l=oBX{kjYi2a@Pmoyh9dGC+@34g%jht+LiA_u-hM7D;VWZ2nF7??G& zNE{Ol2^?ZWnUyp12r(}RS#K!Gs3|w$yl_Jy80z-~^94k$JVFCrM{`jfRlSZQOf*-_ z5s7xbPwOdy**+!lsGuzRsuaqq|%7R2Pgc}ph2c4W3lDWrvXxIaFaSG{I z7Unn-Q`5z)HZc*c^1D3#1d>J2G{m1mfkGyHK^JP-{z9h9a=liRA4uncR;Op~NB5cGvSVFW-C#A*>+ zL7K#~OajOqgP$i9LbTC{>{^Q*REL#r#{v20go>(Xr562yIl1q_e z1}g$J&qkBNA7szyu=Ui+)p{%Uny}M@ZaB$(!Kj00016E~jp~s)*Jat8kc|w4V@+f%O~Fvy?Fo71r76GqBbDCXr6Wqq=Lr^K5IY?V zpimI|zMRmksRWsV=Zl1nC%G%vS$J3#5@EOngCU2L4F@C71O1A#GR&tWu6R0}G{m=9 za4>P>UUv}2M|PJA4VcrTrt*>c^n44}33Y*}z1JVi!R6_x)Gv8p&A7C*np|T7V-bl& zV*x+%0djz`(c{B8FYZdcAtc}E!xk`nfuTeq3MXWqezG)2Ye$Ogd#(g@NWsgYDUVx_ zeus;qpAlv!>PwX+8^!R_eT&n*?Y(_uf$Ct=LjIHmmT)AnV@5KesEe4=5kU;em`*l= z078{3o*O6Zpnky|$Kiz5eofqf)PG1PY<>;%Ou5rFN6Fcfg)tEY8ZJ>Li|%z47d`PKjDk5!6~_1Ert z?RgTeu^)Z4#xP#6qzB-!@g)$e5L0LMO#DVT!S%#gGa_QnCuv=O(}YCIwkrliFXew( z8Tg_zPY+j-rl&rEI$fRj2t&aHL(cj{5V1d>4^cO0+&V#_O9phIMG*6Ng9(Jv89{x2 z1jQCXH0<+40~8bkhiUy0j4%n@epsPM#E(av()3Puw@@;Q-c@|5VELH&X5(ZFE{HV4 zxSw|IjyQ`yWB=Hm#%h3*8EwgQH@pfoyIU#w8@7RtCapU478-OqVrOAwWJtp4iJ=Aq zR5h5?{R*hHF}o$J^ehv!!5s{^{0M(K+#vMu)Gaf#Av#sgpD>6?!iT1rE^B6(8>94D zlW`Op+T{*8V{qj_OnA`)M&5t!2U8%)wf~6;hOHGP`+K7>PROS|O;T{wBD(oStdJ?i zybt&sYML$#3TdnLtqDFojxep)1&t3>2u?3P85b8k9;tP#uDA!0ERSfSo2sx27PP_*cE5( z!K*5rse&t`EwwT|6X$}B6Rc_YD3Wc>i*Z1=H{He+-Aj5i{Ss|oXEPAH^pk>Bd*(@~CQRK%m7CL_ot)A~ANUkZb0l=87NRJD-T-Xu#5UDfreg<4^^)*zVQ%$tX zxN8)57#IUR5`q0>; z(nd&MRIJ6`+J8zL^|1o#%y10uKXk8W`iKa)SL15Aarvny#UMH(Y z5=~m9i6Zb8gE42+s2`dY0?jn|Mk8({F*em5)^FO-N`auk9YcbSFNUxd1e0ha!PW>g z(_8_#K?6?Mq;-e&37u0wbB~`W(UfBZiUwo?1f5PNk`RcCNF#AsC(z&+XTqI`Aif?P z=%5lzRBi>vcqS-@r#>csN7CHSG(pAv0OkwDAhd3Wbt1>GQeG7Vn;G!>a3}xTMNywc`t&$vaHK+kQlBX%_1%ifnA{y`qqB!!1 zt*tPa3WLcum^F~Ri`N@izL24!UJy={5IJ~ zoJ5jS1d{q_G>Vi@1fluP#bRP3CMUY9E#rF2Gde6%87NgJ_lI&EoL zyh2T-oYPJL66gAny6y~MGF$xBOThmpeSmfePuF?9aOwKsYM1sMQZzOtJ|79N8c)`k?J=@3z%3UTPxi*uk-Z>DfI zsfkC~^Wh066E$Uxh!?;Se1F0l^I~T3{WH~mVCkg9nT;Y^#F4;t3Z&_o#`e=CAlURT zMwo))0Ia^uUnumeG|aXLXvnw-1!8V5{`QZCO%{%1;lwdJ&aNvpwk-;apNJqEVVMoy^;N9A+`wQ(*m)m+ZAvkC9KXTCeIX5R~p93Uj+(E zqVjpeu7FRDOPHQ8eUDGxtNz1{4+w}ZIIrlCP!y-Uur6}XmstsN+;A67Cm^l)EAAoW zpx(Jw+sBb*=?*76I8ZQgVq0Hl=W04SG;yLZ!=zl@83HX9D}M~BFFxEH5KCb-fF!(b zOaaga3BR&*{77ms<9?7?0#IWx;SLAfQJlhrKMm%SxME5cyI-I}c0$4%bt4Sx=E=qg zkhY|k85R{>R;)dA?-PDp4n>l`d8BlOW|2qC5%stup$Kj%5SKrAW#G}o{XuZ#cXsnr zovoDctdB5j#bO^ydUKhG8Fb);v@;UNs;6U; z$8=Djl4@Tig2qvEx?llk%yiH+F2I1gT!QW5ps=MuceH@r5XE8QV1oL)0k_c!LqxLX zPY_7RJdH(=-Wl@B;tXc=OF4NP1k^^Pi@I?_KSEX)uT%b=;u!+bw5StVu0A*#Fr4bV zg7lI~I7cG!P-+E50^knqmt==PQinWv9Q=TtFWQ@e6@%j&;^Sf39>(JiF{ws+`f=gu z`h+JKK*m;(_m!}^Tw0&qZdav#py-<5q&c3PxJov zO7hiI`~=iNm|lF{=*9Gsb_zA;S(*!pYjGSQ@*uy|F{iJqi|HK76F0_YS~3`=^tA&8 z90aa|E)Ncb_ssaD3Ba<2&JM`MLBKrqGgHC7I!4k(2u!vY{Y zDlWuCLbN`z3bHB4^8@5<8PsP;>;y~q3V!;E6M#MFKCqmC#?pO(AX>dY8gRO4Snwst zVknKJiv*Ion9GaXk+97MNm!UY(r7I`-|A@(Zklm=qrqs5TqgM2=){4>rddmq@Bw2N z0@G0U`hbl+O(_>Zkb8i;SQ4{L@NyP71V$#srXt7>XxkPld5@ zlzYrr@bq32A|{$rFQCE_N(6$~#Nucf-*1o#VpBgqpwOhka#Ya{%t1_ne&W}zD_-sc8l~qCQO;yi%m{uL3cyfpNKz zrc4*Qz|^g8YN%h5Zkj=#Wi2mF*Lws6ipG1xQ4dmg2UWYdFokjTCoR|3Uww2+#@)RF z6FAioxemsCQMfi6|8IYFUXs84K9Ltr%*2AYF)54?H@oJDETNpgbCa~Q=YQ!jbQ%5C z(WTU-#|3m8#`bt4?hsOxb$^tB+>eTSQXuRcC%`uR`Pq>^R)yuf2dJP40t6sL^1T`Z4b zDbw&ZfeeQland@7)IsJe$F?NUj)QPUyYl}nYZhah-BE(EUWdeY8tVE#IRHPNCfq^# zgnM1&hdZ@!%^32UFydKLM6oHLoKN$w$xW=cbgbwN88DiNV8BI;ww+5Gf?o&tmH;^2 z?Tti&3FL4wjUcz9kRPBaZwpj7Rg8OrU`f^znt=&=QtP`E8%_ zDWZKZv{d;lY>MI0M_MV3nLhZ?T2`lcTCTEapzh$)10h|SCaV29I5hOn@KM+OLceni`3RfgntOVd7 zEK0MvjSq%D_s(BQj~o_Ujn9YM?5hXGTlNLBct#HfVy_<6xZ}4;V3?9?AcQ$!B3K_? z@Oj|M0TNT_1!=&1DA3@1j2~OFgg=2~`1Z6hDcOBpW^dw@7CcZa*`fv8zHTEeLC9&Y zn0L)@XbqzqR{s`a9vU(lqT*VsArQ8~1uG$>Yr}G#W!Y(5E5kpj+pw0(lVUgH1SmI7 zFF8X2Zx{}_1hc)DU|RK&M-JDmCHYA^`W5YmF|YJBLh>q1lJR08x7!z^#1?YZ4wz<{ z53gb$zzt%|_do88Cc8(F3kMH^39l!PYuxzg%uf7}Z2WfxO85vP$VA1K9>-o-jVKs} z+{r(a?!k%=esz_k*oaY~?*y@MKZQ3Gk47j5w}EfMxZ6|A#uaX;(3Jm5)V`QI z5yeg8jzd{IavlpX3NuO~oqU}&_;4X!1U|tiF0`c86;leY3E>7o1N%jOkct_=_;uYX zwt=ovjF#A65U2r;wpap=2uH9@j*62J%df4dn6=H|F0J#q8H+2)|8Cq>?;0$XNVI5h z71i(kH)^y3tPDk07seJWk?_GABlf1`%33KrUyo!9T(pc+Pk3IWkbE%b2oz9@KHQ-Z za%P>LfZ@&E@znOr0fG!-w#zY6S_Pe`DOU@S&?IhO2xsz>f9Qa^{h2 zJt3!;UCk#pp|x}f=7^@eE&^f~@ApR{PMnRK+p>Z+l_%M)$Jediu%LFM%KoRCyK&wT zpfGnM9vq?#Ap9YY9iBm8fih*A1~0v`Qi9*d0w8wm5%}{$QJkR9mCfhGmCzn2E0>_F zq@DTa0v7JWA+IbB@8hncoO;p;4y=<{=;(Y9DM0cg>GJJC6*g`tExF#B;i#|+m&L`L zew^Rs*GadUKv_$Q35@LzW00@%b`jj~@WR;<^23tU61OOJQ}gw2&SoQQ}ZQ#Op(R$e_e^4*gNi5h$?))A5m53}aYd zVrc&fP3QL2)-TBZFmnN0k7{Qw5WvOdcrLI9u@td8{SEzmq##ctW;u=z`*A5s*OFBK zd(smHnwdEKf}~p96LM%v4+|@aJ7CG-eY!w4J?xAk3XAc=J<}qQWb|Gk5FzM<%U)uh z1bk&^f*6L)|KYDb+*~x4jf2M91#UDV0zoGRv$5TlTuZP;ec}b*()jqRfK#7v`Qq5` zlQ-5!GFxLxTF2fLXd2=%+}#(zIetDhFpe!M$*u5%K!x!X#C3mu+zq7D)zj~nDLDziTh>RGMD?Vo)bPrSAbLU~t ziMzo$z8H}e0&AM0-e4(?nL#Jcu7~KleI_5fH<~`C^}>wxNJGjdKs4ZTRX4gzm`flw z>4a9R_*{SU4!zv1kMj1L``hrdZ|jWx&R60adzjY&tCo3hD4l}kTg>s|Ym2gQIZ;fGj;FS$ikruMYq9e5Lalmd4=Hg}#v ze}v7I=^4}kC{eHgq`{S(NN5tPB zYyI9W{{9&2_Y1|}uQ7k`k?@bUelOv#wtg?+uhM@Xb!OLKrTC4+V}e zek0W+ZG4l+mY_%cMv~wt@f%5kZt)vQf|U4;Bte({%^^9x*eQM^NzfsFBT2Aa{6>=C zNbwsr8Z^+!0QJ$PbiFaYRM9NX1Jb@^?2x1y2xxkw_rMIh1V4IJ4EfH@jM{z8O z$1t{AK-}cC+6XB~PI$~MdB=g-iA1<>1&Yf+xI1g))3OL%IL|E+jsf91B7B|*e=rG8 z*Tto`Pgw%bbvj*#)H(EqqMoe*{d%IO_itrlHbt@Ts8rOSid6d<#JfQj`jA@OCg(8) z*EA%S1K}h>c!3ap4}>a+uAU8qt@#lkL#;!=&@#INV*_CZXL3&1JNTH%k+EHm!?E4q9*+TJQ z^Y#)pST6ERJ&u0s?9D|w-l@P(~I}QEAjmO;E@h8hDZ(Gct?<#8v*!70^9?exJ>yF zMcETWOj8L?Bp*N6N>{q%%6BK1?{9!Rjo>~excPuHT4}FH2>L#E#O_1x~K>DvNmj#tA z6`^Gcfps=vP1zmAZGu!^0&7@_I1BVtfLysw*_};+U4eBDVSP6W zNZB$GSjG#e3kfPc3B?zJO#1_Bt|Sx7w8|Cg2`OfH!FyjdeJXj*x}2 zpt7YRw9H|^x|pyU_d#(Ou#8bZ7g$G%wiy1LI^(+ z!i7NCn=KAq`Dw9K25Svvs%WZ`X#v2c1n}zbP&@#D{dQCqDE1UivbqhQAF#_rd<#a7RMzZy_`tvC!;tNq~n;ClP1!rOzQG3iW!E2VGk|? z{S`$2DA8{RJ((Jcy^rYQgQLHa=od~w@tH}#g6KWj^kfyTRk3gqX2Ln0UTC~0#=1m$(p0mZdo}1I7EMA(LjdI34q#0P#+UiJD|9sQ^ld! zTRTaZ!D3xcSbwNPvBkuy=2-0ht9)2+Kaf+vVOe4h!$u~XZY@N-fnc8}*jr54F&q|` zNalbBFm_H-d8eNU@*9afP>_{K(TL)9lf1Fi8oYEYYoJm4 zIa={H!uSVayao(DYNvAy*6bz*^+j$+d;y5t2_iThg@Wxd9~Lt#2)0`=VPq}vCCL9m zTEc#@qGWHliv_^+h#@HKGmAe==rP=rh|4zNIc9W6^O zoJBE?lV-oJqm7)QpT7mxorLukVWoh@_47f%qHPBR60x9{a-sB1$ajFci=a|7QCtrw z3^J&wCN36j=2&cjlI$$mf;8=PUHd%{?k0q<3E^Y2Vh)aw;788j$%K}bF&Bgn5aB^hDBb}f{1UJ-rhy9M zsHc~uS8|r(=9TCkB)SWUZYah8DJOLX7&hi3jS9^)AG8k!bdli;(e>D4UoCcMPYz! z)Cz~uMra{G9wCsE;or?%2oUngDE7lR`Jp^Ej+6clADaXEtA+;BR~>+SlpsGP$m;;f z_0?t$X&`Vs9cX?QI#pA5ubmFK9R#=7h2j|#?o$PB-{la6s`CP8eCJm?W+9( ztJ2xfr)f!N{)rR>uyeH(mSySK(&nJ(H&qGVG-5vIeZaZ?cq1Cjg!!fQA5ys{pVU98&7~BF(;&3qk4#>ZP-rAWy^Z`5^e+(o zurP{wCjHd@=}n&;+eUr?fENiMO#t-9-~HLrv>yS~^(TPUA76w1C8EDFg5oC7>q?%r z0!;t*Hny%ZZDCe}8kDVWdX(TlKzW%^3Zf`p1d5wUF^l|1wT6F@*Pri$NR5qPHR#5| z@(R&DJs-tva(Hf1mW|K~k1bM9D58-OFSf0O+h?`xLSl}BtlhnmITvWJ5?aRs6!!tm z*xAG9X`e0mvm{r;2)IiX?5%qM@EQTsABMs zRD$V5hK8Z~DFEmL-0K84aUqJm0XKVDnQFVXNZr3!jRWx@Ai9xH2s=^4G!|6CSOJVT z2qXG?6dQn%`Ma{LOxdMZ8NRMqy|7rlw^)6?SpA|{9a^G}FHvWgsB_@QXackcex9Dx z>H)FeBy8s*6lVh40c11-5hy+e zTp6mX*quO$(S75;gfWa`yiFM25{4a)VUFPfMlcP}4pwfVK3PH(bAX)$(At7xnh6jB zK!V;^Vb;L>;9w}|Lut=lTZ^u(chu2}fsu|XmLdLzD!xPLFA(|ypl420myH;*o1$nJ z7O7~jNU<`%*p@1>9aUoMD^XXJs4GjK3*iTXr|`mYjo^+@g5 zky>V?W(Qdla)~3qPkj1d7X z=yKAkD1l)ryQ_KuWL*ewSYpQ7Wr_>q zoB50+4o;dO_>JX108bvWp;iT-P2l=IaZN2pQAw^NY2yc6Bet z?4FU@ec-f|b0SsBEsTLH3rwTUslum${fESU+)*ehYfyYa6)uGU-NhOz zTy0k;vT9GVtMlyYe7l-vQk_?-ZZB1z9;v+n*}7%f^p{Bq9q!E6AN`7$rNh{mhh%3# z*pEoqVJlFaJORZwBrKG^hJi7^SnV!W8%wq6rRt1Qbylfzda3%SQuXRmEnTLbTBZ&w zR}L;$4=-0g8>#*PsgH!z`i95Z(cY34PiirrZHkYcWk{l|R1467SSJhFPy=fG!OVK`r-~*|SHe=s}j$^+=ZyAO*xoW?L zZSYUx^}FLxJi3RXj3q0vew1o=5KvtX5E&b6??~R-Z$SPpB7d343nv;IFpwV%^11Bd zm?bIP0-0Lsa9|^1*eWzC=0UA|O~j8Mk77B9S#Y=)#B+3FHp$V5{gHn|(C8=R1sT05>d3^I|s(TU>k^$h5p~60E{q0rqbKJ9aIK$%w{~m8{s;Vc=Wl zkQlxd3`4St)XY^F-~S;9*NG_p00OP@G62vW0phFtZk+9I)&a#89g>>Ie?L4dDMJ@RIc?t^s(a(*~1n3MMBM z#_2`srA6A+Me0*U8npYuV%ySU?Z{%a%06tny`ZsF<%(o;sdj3qhS__QHG#F(a`lhp z>bd3GO{3KEO0A+&+qF`wtklL-s=HUJdsS-FDz(N+ZFZ%0K&5t2rRJ&BVwGB3rM9$E zJF-$sRcbwGxqGFKV8Pp*9(Wm*_=zfU$yO8}Am}s~Jy3WCluNP7qrevd>Ou)h3{mGG z-Ur105#k|dqbS-(>QltKF9+Pk{r0H{oztf*_WdC6eC zz#js5W`F#fa4w4Z0MFbDvq!6(UtEa3yNVXA0xVissLd4%y~$ND*T2V=tJha*H&$xf zE47;|wOcE-+bgv@Dz&>RwWm?_s~{$KTg1EA%r(|^QP{i-nz(M|Uwn-r3aNLXA}3Hq z_P!Lw^nDfO_shx(hHNXA77DCZ_HDmeAalj~MQI*^G;2wkpDsi39;C^(gtB^8V5cWAZeA*7{K%y@s!;QJ>Sok)z1z8b||Q>4L%%z+<)^#ZW+ zD-;t~RCr9AP*y_bGk~un@RzPZ5e7IPc`pJmpV{9Z+%TS$Kyd2`&U-D2ZWHcxz~wjK zWx(vgoX3vmj6MgWlZeqR#AquRWtJDfmYZmGpT$Nf^r6#Jhc~!QCN*_LB-!m{%obAUEFVLIFK{FAQ13&rKX_!iY4TUW)H<0d zaW?s>0R&rB%dx*S*k21rX%|*$7guSQR%usNY1^u_JF2w1A>{W}H2>)&g~O9v($d$_ z%YLHp`&{P&?DR=2-7Ik-yJs^A^v%O49-GFapDNOj?1zlz<^V-Z_#G463}Bi=bXrV? zX>}@Ly!r@=|8NX!oQ5IiUk1YGpY!HSv@Hu4b+zmc_ffeBjo)aLvbvSGqIuQOE~lj*!o;3 zuFua&h#&T_TaTDsMW zO0}|571nLFu~MB~sUA?N9#p9s>$bYIQthc!Go!T=Mr&(FYbTD@HjUOck5ZPs%T!8?^VK=V{MXMi`%t?T|3gQZMWOfWwtZQ)bq-;gGZ_JMyWrJ)(Wb% zA=O$*wPxQ7^61+#!98tm&Met5lYYUs4*NF)7p8J1N%YRwD4st6#RKHmE9zC$lZt5L z_*N0Lq%CcyZQ{jc>fY7rKGo{)Ajxd0&v=4SPJbN;{H=sP;Rh67A1L(~@{~3M=l}ra zpMD%PtHTO3mb2sp@>zs@+`my&B5cMx?4aBZ6JQ4FEC%ZW?AZkS3Beu+*vv61Ts7Dp zLW$Z=*Nx!dj?_?k(o;@L9Sp|j z5aVVQyV(buq?SVR)XhMe*KbSds5wo=5~=0``dos(lQzm90h%|}e8A;f`16Ae(@e1Q z2yBuKU?mO%?9kkH7jXt^c!qljggl>cw-RnWa5Fox%4_?dM7{zViP(-@&bA}Zk;zrC zuz1K3v>h3WHGPa-<0kFi#^TObxWj2mVerB&rI}k6qsLUJyA7o4Y}lrZsnm9_)b^y^ zN#QioY_?dXZBC`yR;fN*tvyn$JzA~psMa2<)*i3co~+iMsn(va*50euKC9Ngsn*mn zTEQ4?$QaE&Mzya*Q_Nl7-hC)8_e{?1ZfRv{CsLl8;W?%`NQq(>P|ep@qvm&d6lDf^ zpN}e0Hg$PXm(R+eb1 zAl@-LuHl1Zc^$t5cL{?-?sLWgxH`S*Ht`A{t0Nk_ltCe0N{EdUfVcsO^*Z9o9PxM{ zGV^v`Du$CsIkjDcw@q6fu;pmP%LuMvcfdUjxK=bG?+(qr*20Rm^s-~jF1(yLcensl zMC}CX<*eEhf%*rE@?C@dh^+vP2p9HRF6kzqQ@?AwZ!+!^y?ZDZW z$#*Jnyl~L7UgFMmc?`k74)3bqPwzoZuVzU50_g)FWlmA|9%0)Rmh)Rh2ok86KPy;Z z0JG@y66g%_u`e8DGcBX)G3v!*wCl!bH;vJ5gBn1#75TexR|lOvl^dEZ&CXygx!lDW zw*q2ZLt>m!4>6`9R!Ey(#s2sh@?Qqyn6+;DsVa_CsKPa=x-+aABp}l&>NB3$05Z-800Jmn@u%2uZep|r^9Z#h3K6Jg1#uO zC^7PouAz{zdSr=~D$(o@6G|PXXch`zh{!oV6DYS5%KHZar4}f4Jlv}i{bQhKq%_L} z3>Hvg3Ef7FKHU$UtnI=gi?qD)1q9)_PfOI#OVn>m)Sj`L{ZoV4{H`t>dTKKiHIn995pV8l zVec>%YUZSDk6#evFC@r*UI;SjV1)T0h7EkbE>YKjAk#ea<;uquVf zFL)n`UbvIQIXDb){B!W>aX7UL#zT`|U4-FQa8ya*ijrM;toL}fRx)1^a=PIt8v0|2 zs@l~;yEfdeeo~<_dyqD}=Zw|PAFJIkR=atucFS1p(Xr}|vFb}>ZLg2jcA^sB8kI1X z!(ZEkcahB39}bxxn2%4NXs~<{!YbTUG~~cy3RfdGkI=fiSoIXE2$82s3Xdw$uq2@( zYf98(sUmQCsq0HLtWc+x4Ch0cZy-+^rEVCd{xDYiajZ=nrwtpY*~e++Y2r_C6r%^auA8mApNPW6vdn<1x) z%wWpzJm%VqK);CT&cAR#?IBnfHZ40VWm|YbkAv~uh=Bh$g zc^4Y^9+JH2cu4+-W<}}3@E(rgz0IzQ>@jFxAFG}ZPQ$Y~>6za-jMxzrJ=ot3KKBxz z;0fSUdAOpqgU?X#xl!hG4LJOZhJUhWNvfyQ96-VwnAw)V;6|a->>&h8NXpA0eXW3Z zErhs_gm9k-A^r{_GT&e{*+yVPxRq@PuPlP<#+)ulm)KU6*w&MZ(K8WlB~LC5eBK2Q zkJBC>r#%H}FF|{-RSOAht;>D%9%d=&i?3WCqqXlR-osA?@9P&S$~3GwwnyyhqjvSb zcFle%!O*gm2gX{u{sox75zNFbfO!ruODRCLiNQQ!SDz#>)xOEj7Gt0p4d`%S#)<1O zFfM_C_yB>~PXpM(BdpbUCNQp`eon&N?;-6YN(Ij>n%vBMqBBk;AvN(Qkncf){0Bi^ z1jxD2-b3L2nQT{Q+ts`6>P}wYYfN}^(ha}^{PG2J1^FzdWEnab!QClp|4R*Cj z`9Rsj1j%rc+eGU$f z5Qq2hs-Vn6LM>+4Rj0^dJ2+rRkhgpCq~4&VjP$hQFby6h9$WCrUz1&tJlw!2 z^9_o}h?@s*gveY2ZcEWGh1V95`x-p?#IWCLvUPT?&K=Z)U`Aa;RePM6rE%waCfKSd zHB_}bdDZSTdHf%$W}P9bH?mTwnqQ-Mf;iXWD&Wk?;GDSv#*u9pcda70UTN2mKm<|r z8sBY7gLD;Y>R773#wP6`Cv5f8*xy5nCrOIaaIHjUd7GjfipDPBjji*Y48BY!O6o_- zmo!dB@`TW>P(@D>XzQf_9i3E^HYnzTlgHQW`~5N~o2^!w=@~$Mnov`h1GO8d zi-B5*?ZWXq$&>}|>us{viLq@Rk+BwSiU4=fhqkCbk|nAr?S%m_`zl4?I-Ks-~q63Y2Nd7e;uw*lpGpi~0|M~?=C z=)YqO$K$!c{RiPz;*O?F!BSm1>Idh5(1u=}T<9^LE$0@S z+`YB!_DL+my$!H05$tQ*0b8@oTGNjM&~YSgorGiDAVIm%ItWI|LcqLCFqhs0m^Q%d zPiq7+>vWMe0?C80ZBHlOuti?~c`1I1zyinno)*O{YqLhJG=w|?=&!KWycy`5fKFC| zwzEvV3qqbu(D~5lC>>*h8$l2$*3W*Si9@NyJbe#`Es>xw#^BQeKCcp=HMfG#o8Xgq zh<1~(J1GJH7d}{CHeS7Fym~(vp9IEOQn6I`wzryxY1)xtCR9pFEHiUH%!8|7Kg?ji z2<%@Y_Iuv}_Q`hSmobmxTJWJ4FAA_oC^xc3VZ2&S6jPzm%F_(I*9q@(!aEswnU5&C zHq;D3%qyqJ12UrCfzNOq*x=^Eq`-z}7K;pM8k3EVG-pHXKY+~}jLl!c=Ds5pWghho zTI(yIPO4#zWV|sB7wqyE;28$_pfc~)gT7>xvAo_V)Fz|s6TX6Bybo`%J5 zuG2WmQiG>2szk~#kzGY(O(5eZw7iTg)h$qwV(sgff_$UljzREiu|WXajiqC`1uduahz6h9&@ z@~2T;QRpJ%Q4=q4HPH<=9}}As9tE4-IxRKv8em?diqshu5w4kGN`gZ`NhAUCPXh7p z0LWngq0LAER%}QIJwXzE2q;g=vl`B*>^Y?inWH8~*mNQ3R-%6q{vnS8|2W`hdMS8{ z!Gr0mnOP@B8nTA?>6xYKc}BaK(*lWI$UwOk;*3#WtDUlEJGOUB(W` z_zB7Q!LyJtn8GK8(jNC>Imts#E-K)0M(iyAT4ZZ1EnHKo@r*7=illDHcu5(#{y(Ll zA;N{~Ue($@)fz(L7mO!$*j%F?UZXCqQQKR`&8DjGNC1@!GgVMxDLg+7u!;kNS!{#(TU4mJv6?K*(0g6&s`Pf6+>X=H~UX`}D$Efd&v8|q< z+TQ^~u9dAmfeWKQovc@>9SpxDhTc!Xa7G_KO(urJmJvhBct49Jy>Bnoz8C{NZL@z0 zJ`}b>9JRZ>gSma9NVRTRiuFL}tmkx?8+lp_^k{7-_e^A4`<^03NPd^|~e3>bVv3@-f-80@vmQhQee?`o2~ zE@g`s*Q4w#{V6oyn;Mv|3Gzvxo+xQt? zmEFMTJ7RRuPhj*%FpAi6K30B_|tBd{z zR(-1x`NPan>~|5iZj|MR8?@@=jH9d!&oZ6>2s`zKA@0(+pE<`3mR z=eHwZJXl+bI=Im5B?IDJtd2ht{13%||Bm2saA(AjU9b=3M=O!dZtO8MZaTD7tM=Av zW;$4WbLWulE^a)04i^6<7B`lI#rMZrvnU9lAjw}R;fRw*2f_4tA22@=Om78X_Qye) zgH7{+wizVVT!7^r(iVR1NH*mB7a07H7>pVL21kHFH?{N5GN$?G6SO{}CA2{gWP{Hq z;PW%_IbbCCJP1CSWi)CR)HFJs2UEc${YlrWy*$;+r`ImA*)L&j@ zhZ!RN6C!3P9q+9%5HXZ77V{|hu(@Nw4r`tJrjb)%O)}@1LaLLer!AS~~pm zgC2JK583=BxJ~jzB1<-cY`+aW*sa(fo}?UjAlvfLQ+7PHj-DDZ9Zz-RDO{+esrOG( z?GyGmg@itqY_iV z;uNBocpxa=F=aYPu0$5aW}>)>DE2tnsD)Fih1tJ#U~noue$YX9ybX`@I(VfzNRMx! z$6ut!FEJkv$u%(LZv^}w>G`fEJpZ~WO-z?wxy2TRgVZ%{U>{IZc* zU~oDSY;=I&4;gQ?E3nj%hdnF|(X;5`Bi(rTOcQvy37j?B z&ZdXIqlaHGk&m*Fv&PLi^z;@Fo-RMtkh#x=aZAftyM$l>CcO}tW-U^ys&q4Abl)ie7U$x&Lk;Qwm zsIDTaYuZ3{qDXa{M71E5O?EYr-H-&?*CN?pB{I17@CGKdi@@^~NO}zs-@F9Gt>-Gr zWYmsH`k+LN#GAat*AnraOF?|6Nc@CE>|NEmtfgydQeaF?bRE&&zYMgc=PAlQMiV`s zmljuSWsz?q@_XAsel*C<7M}cjWVTv8W7gV%OWA|;hAGRHcttf53JK+uZlDxj$W@5C z5&iTQ?xDdEBGcQSz1?Ln<_#zhR3NXAfQ~;2Ku283gGDyJx&0bf>wHUB>)@Vd6ED{u z=XcsX4Wt_h>F$j{8h({!jl+hvzfDrrNhqQ<&LCimKLn6J6Uf$00GSVvTG-+mHnAiN z&JpIf!Y$lrjE1!Y;z!`XY+8%M zfHp0c_c7q#Lim?#0sdFON7A~6y`O5gu`7U>-p#-H%kNf`Hh`}K{#Js&{EvY5U!y2T zBdj!(k1W3NyMehLt8m=4x}1$Nfn%Ow#%?|O2(Y&itm99By&kYe>Sh6O3fR>{GK@6N z2m|BnO`zOPC}Yk5$`3$caV3m1xUfhd9CGbv<0FC(V&|hVsds|kUx?qLGr@1dwZ^`u z6m0VDD+MmO8%vJ)9x(49%=ZX01WfKC!P!7_Jj*bHAJ9zb_rc&!VsO+}F!&=F@Bv)} z25dkJ=z|*3?*Q;F0xUa=S{;Cy%gCEIypv)H)bxCC@W|THprJs9FhU2vn!vupEla?p;*t!;WK4#!i(hdcG*{>Cji7m{ML91w?>Yly5M?(;^KZPaK5 z(H|Or>T4*p59rf(^hrU&m}Y;4okY$P&>mRyN>TI&Nc7~*5dF^W`1A((DGG5+mW#Nv z%tjH1*H|O*$Wu15nSN^+yTuCbe*aI+ogMG8NOPNU)}R9yQAEtdi>|O>7|sYDI;H z8TAUFUJp^d4tWUm8h4waw_o6_@&%uMEw5sc*Tcl?L*nH(dA$K%{FXMIoiLFlW;~bK zh~Hz18wUi!sKq0Myy{^fcLO;ylX_=&L@zfNsVs`3K3Y`pSdsQjQNc?^xCf=j^n>DEJEh4$krx%TeD1)=>YZ!4Wt7>BiTG-^E@ z^^Xy=BX)w>HFx3D6IeA1;Yp{gG=$^=t(B_%>`GljTm17V=%=`XJ{?RRCnnq9 z2b0xzTMGJkKpjstfr6HhB>l``pr1DY=m`Sa`~iR-29WGcC&^9$oMCZ`piszz*F3u5 zTfpy0;^+Jb{5}J}|G%1tl#co;4d2hHD8}2gH2Vfrs6WkPwHem_{C2abc=T#*Mzx-z zYF+&esx|gLL-A~)rpp(O&T7Z(khBhaDqx=`*va1l)(6qUb9oc z?pb0too>>6=x>U0ylL;*>`PD?4B;%`8i~pQgc~G zp;z-EN{DMuo+Dm=rA)@)12{<+sZHEL&q!+evo_{Bvf3Sh) z0`S}wJYkj*Lp$ykhoP8X$>_Jw*E-~#|0hT#WH%!`L4ZpEI1fi5Y%C*HeGHgSlrsIn zL31?+oZ~RD|3M;MTnL1BfM9IJu#d7gkVxznSm|h|ao6c*VE+OUuNea3sSo0nXSCDD z`8ne40&)HcrHeC&sf2q~z~e>Y;T{Sehk^&)5WpJD!x`jTzz59%*L@O)mx#k1#33Vc zSO^aJjZn@Y$5Vm(G>4-_UnaOI!vJ?J;L0(G75g6;7Pr+*>TFe6=fKF#PC-Nj3Yf1D z<~4-*2{8FC?g$IBUQc0!NP_{b;h?V)sDC&>=RahCE(0jdUraE5oPtZsuT~UuOY$p1 z!DcCk*sl?X4~WCP;LwH?bKLhr874?r`i!E~s=X5oNk3&V4AXHOiC^Arl9?5%fYGkt z_&Ra?xCk82co?7li*?pkf>R@~>0t`}Su*Q!n}QmvRNtg%ygH9fdAz8QRiD+mzp5j@ z-+XNXlvZX0_`X4Wr*PJ zU^AD~W_u{2!F-#IJ1Kb;$cMCIuuLo;lipjzj4oM!i1k6#fc!2Y zZzN||`U$*;0QF#*@M{1Fr0#fMZ+D-OyBua$2LvJDFPv@~58(F*c-ig%eht8xsaVu) z#Vj9-c6hH8>DO>IXj4~_CKOz8s2z+JUbNv<&};mJLj|PF#K?E zUlQs6^iEMD`?diLcdJD9Ey?i3-jLz-rzo64X|#SgiiCU5^BjJ=5hsp5Aj;0)f%3qo z=|XHIiq@5>^=0ZEvc3@j^<#)hH(<^#H_fG!4RXVxh18E`iN_anmncnWZL0yk5IEsbq7^bf!3 z?XePda=E%+xjL;}jh18gj07YbJPXF{MFlU9(I$@7_8hBC8LQ16s|IS+B{;vObx^T} zij7okp<*i)_9!G|<{b?wQ8sCo2x|Fe2>CGy`KK8W@}uYQ=>oFqMnFiO7w0ck7n388 zU-m?oD?L)KzGisz3{f>)p77=v8#K>@5Ja`dA&5D53_-|O)}J7^LXdxwAd?&rr05^` zlp#S%n@ZIX@h>>NOuc}487?hTr?O^ypj>^uT>WT__SqO47LR{1&bXn7U8`DB(lg-`I_Hv3B z!oCpWQxc;t3Nem;i45)$Lo6Zq(h@-O)ZwN8CR&aF<~Nc6rh2CNaW5EtMhvwC7+(4^ zKAor68~I}zuR7;yZ#7xwCdcLQ&V57@Le3Axi7LqPImyvH7jhi`3iB)vv7zYc}*mSWUXL1*I(K+me?*_qNaN^)f zAV}~vN$_$rB=`&xWY(i83azF%u2Srf4W>X==paWP>nlkdbVr^6U0aKbdHP-6jh6Zc zuF(U^@01aKn)3=D_N&r?FrLR9s7h9%Ika89(Mn_$zlB!%4~g(yA4J&gLwu?y-|8+W72)0xgkMy+tWPT< zWAtd9qpGU{cb@pga?T0Ys>wxAfPh#?AwZTEL%_1SF~S2P zA&DszYXqf>A}AIV5U^mSDS~1_1S$5qi{jd_Ef&Ns{=eUuJNM4L@4kS)|9(C@hUCuo zoH;Xd=FFKhXOI9v?uM}EDHxe8eK@{NPX#tTQ&{ySR=ojL)uyLjR_)zW*YUoMREb*G zvsU2@)Or-P@cthBN!MwDa%}3l&h)VT(cMt73J5n4;Y%Xyb?Dxp5&D$YmO30Gv4>H& z8g-|#Zrx1OP1~z^o@&;WPljm@%xzu`YENTrF$=Y`o!Zl7Z3Gw&tBF+`)TPqZd78<3 zEdZtyU={(&0H9t@oT1740H`1Q?9X8RHnUOxVTXa4vVK9$z{(NOFq(Deodc4t1L>Kp zdOxea?NnW2E3jr06rsMB}wg(q?rSbsC7wA(3n5fykWiTs_kQP%X%QrVq(f zGBymVCIjdu0$n&C|N1w8cr&UIoXofxrQ!w1+;Kn*cGa3x0>TAsevbl~n@OhMZ6K5M zJ%0L>jw2is&^cvX5VaqvQSL;nnl~aFvV5Kf-Ye^N?#NG+yeL<%kGkJCs}>gYl^lb# zV)VUpo3Eq6*Ie?o?oRNP`-5fGfh$CiakW0BwK%P{$Z9Qyw+=^7cBy80GME32Be`_l zeI$De$!6RQvcG_=v`I1{=~G-fgNEh}#8HfViqS(7wn=bHsan`VAg6)EJd)T=6211} zr*`m_h-2lA`Y=wkRj|Q)peViwe}BAS_)U2iA~Q9fPa!ha_X)D7lTiUb7y&e)uzf`X0QBHfof_50SX>VRLrS@exH>Pl5w0lWCk} z2JeLIRa0hduqSU617iW14{ZkXkN>JqfLkd_K_^##lT@;vuq1UmNzEgv_Z?DrGBEhC z*(4OF7F%jqs>M49+vz31hJM3OM+TqVDBj%}z4T7k-)q|$l^ zs7(4DKYfU8W4O6G89;s&j5_7w?XDw^pl#bAKS#P~mo7S_i!SNnX^yxTFOYx*5KnV^6}tn;N#^#@Y8GHBPsPz_~`5~A;^S?o*B=?!Bh@p<1Pi7t2W?j zA-US~7P$KTPyF;VxC$ZH9G9j<(t&PK0)liDZ?0aUsBA7fflKK=HV4EZMF+G1Uw4zQ z$iKnYKmNi`1&rTGgI7jr46ue+9u?X$KP5@!D8P$u2t$%t&5@%Z39chthQ%3-jOz)> zxE|?GZi56|crpR(N#oySuiJc5p>M3;{m4AB&S2dA&WP zT*!Sv$2e0lso^a#xcTW8;{gKqw;cezh|nXx1oT5Nst~{!YJ8oMVhZmiT#5JW*oHG? zLwbV%j>F|zDNle*5^9@yegSqgL1#hjKyNYWefl-%9m_QIb2&d!WCnT;0%vkyC%r&V zku%dp?{tx$&O?NLl8GEJQ537KE{lzi^{uWP7r1i6VQ-}5Gp0v6*jPd~e%%W;nuWo} zM7hpMU7U1IN=UQ8xXIaMBZTYKn5jsNgB1?Bahey1$ob+TGIJ@5F_Nu3uqAIW76;oB ziLynt2SfLgp^JV3L(3b1q3L7@GlMtU7s3STXoRpBt8h?Fx;2>V4CD$ z0636FWZ$~-@|^0?HD3M2S2T>_q7GBk6l_fLp(iq~~MoCq)n~2&c=^NJuW6 zJcoyxP*&1B61`ng9;_+DJgQRWQEa4r+URS%cd!I#uP$I^Ia%3<7j5e+8-tY#(ML%~ zK}#T|3Bqr|1cnPjNfANXx3!SgotwJEm$XFLa@R}a=v-&mjH{wE&@UfXo0@_$2J{Kw z?LP9>w;6alttogrn!F(=ER#CD91&mAZ^RzLAcmxn;JVW>qVqB0lw-uH$AsR`7Zds! z=Gv2FYdwptl;g`Rwy}6!inUma6+W)g@_=5yLJRRWOqr!1sl>?}p zY-f6KJX=Zyf~j~pU{(<36T(aej5ofKT#W}{LnLr}2M~A&1jb_F z(MicB65773_sGTY!ww z4(o&ghv5v>7|Q{hr~Vy8`kffOXS2v>F@Qz17W@YXb{VZX6f4Di23oPI zxv+DaRf3LO`d!$D7k(SB7I=>mZ(JtuaBPi}MlSsH(#s`39IXblG_;M&6#kNCoY^##z{}WboxJ@_He>2hePslvXpfe zSo&8B{OW$N)c7cs=ms_-!uFBd1v|Nc_7gOCc}AFbxG){DM@3XPvc28T6QYIe=X+#l z--&-qw~x_)_z9}6z^no*E6K`+K49g%quuE_iKtPQoT6SRbB;D@Zia9is2Auw`%^wX#QF@|J(D?{FB<~xqYo9B&VQ(dq{I|1t6aw;O*&_OTvkKmuF}2&`BrcJ(YORixG!UX`v7pqW3cN~wQUKKnE=F% zXqpU=rwH=iD1aPxtUy-L7J@#~XgeUtl#0?_XPX@XfNEoIm`B$Gb3HMy84b+SfY}Z< zWa#8{xvvq?4jy3BR*X;OXa`HlU6gIf5Pk#U3&sF`u>sE=lZbZK@YuSS{GZUDCVGo9 zpf_*lGOy52*64A(!j+~(fuv%FG^T;wX9&2PfQT4`tTFByMN~E^#I8Vz%J9VyP8e^- zd=hh6Jj7xzi|CIi<&Z!kH9l6&w<)p*DBZxRkS^KBeIAok4qB!@1Zwt#z{aFKezi7tSvrXxwi%|IS%h^B}R@x02K+M$)z zxIhQ-@Ib^lfz`2gJQ_I>h%XTF#OXkM_5{cY&jRt(HTnetByoSyj=SFXJ62d?}rxjFYM0qB{0EZfvr{ss%L?X7m4%1 zOyKke&M9BC#MT%h`b)CGvt>L>o`{)rKG!`E%xose=2-xF0U)Q!KHm$3eu%VYYLy%Mk>lSZBrRLD%OBx zXf2DUSvEqC zK(KLFVIQ}Sl~^zfoa5F2Ebqx{A)vMs>de)E8Um;zWrT0JhAPf0bU_M{S_TRvjz0({ z{yVKM$#D_L>>!!vNoJx$W}GHd6dPAr8$*N-^=%Ld^SpW}&~_57-x{DT0h&6YUntRr zRM;Hi)HZ-CL<}%g#NP@nUL%J2fPS5u3y^vdc<&P#KM705MX z1)c}en?xG24oFuzNJlTX`$i%i1Ozv-t+xmgdkP>s0fH=WLSz9$MAI~q448;RGG(<3 zIByfD$$H@Y1e~LZbFRd>O5+rjR*WpD7#W+G&7Ugq03WQ00vW)GeAOHmIBBxpA898H%rhE(7N#0aIlpc*ve*mO3a4{%JFEP1F$ zw1D+*a`4nf|^-GDO0Pr3PfSPEKZuT z5e_GLNIfL=JhPZv>4*#j$^Ve#vs*#(tTO~dqlCx=433 z9RBrA3uULN7{y5xz#JA_>%FSSW)|Z);yH0}(GvegTqB;Wt*a;- zfe=6_9b4y=l_PgZL(F|h<}P{@%$;-=n0t>yCAk64E+XxVSzN_}Oeub%o$w>myRU|3gr`qBj;W*hN;9=+5b{ zgCgy#_$3jmBs6yT{7&|vIQvMINACyNu}=wspmneo)3U0Mg2qI9u9#m zEM_l2A)71({SMps0gI1Vv|#TaC5Hgz@Hg27pY~f3F1LEiWZP>9*04S%$F-k;<5oG~ z7};uCG)l)=0_rl0{hMJ|(^B-+|sh(CaLp zm1XbIcCAQcuQa4s0e35KKO=6<_rTo(++N)A(DoEJvxqL26mqL!5{)RWsZC%OadQx0 z=^2CcHc=W8Y{;mImK-)vKm-hi}KhXGYu%MJhlxoaI zV>i~-7}JJ5DVKdGDcmYReL<*gKLYC5zQQ^gBCw%eb)f=hVcfuY3IpRyqP@8vXk|b{ zM`^}5&1}X7SGu|s_M1CX;t;n0@hc)e`4bRt@gj20+K68D5C`)bV7?~IT|Wb6gBOOg z-X2Ei0v{4LKk=#w=8xAdGy3{X(E5h7uKfkHJ_W6O_(Q_Zs0z_i6g2Wg^m9)O;uJ?* zJ2nFQf5bliH(;0a^9cA$e>>vw{(fto2gSXlnEX2^BI{rSt=V;->JPO#C?%p4iZU7l z1PQtEMoKq$IN@~wt#3(d?H{0()L&Sq$!*Vv3U^Q-E*(EPxx?_N3Hty(L)x?70qHv; zz5FMT`T(hwL>hiSIm8AEdkAe~pu^?uQ?&(*+m1AEgS$t33}oRTgith`xlGAp&ZmyLIJx(Jr?42iI;-fpQLtcOHi9qXzvo=1q|}@`M|O@jBiIreRF^y=d}XJ zivT$tmO~P@i0={SeF?@N?VDeC{%w+44r&^L{ELu}w+3XRb1|`gpz5_Z%+mCyg(Lm| zZ0Ju?Xi5P{QE;IUQIeGeReEe3VU!~w(?|%vL8|rVfNJY>P+eUFsuQ?1#RUx-_wv3M zvj-^EA_EM0LH!!`3GMc<6MY%t?G(gsuU8xmdgak2OTxG z%28`9SOFW?k-Oo?gS(T9-PPf0(urPyk`|S&&a24k?g7TKx;n~*dMscj6Q=tKfSCvw zX*i$!PciHP!f=>*VBny2UzqbiT?VKrgqnULpf&)iCk;?O6f2@+ii?%Qj^kHgc7}1N zdOfSYeG;l#!_*0wJj7z)7k14PD&lSpIlE)jyBq*F5a7m>0dN8Ux^YtS*os?3+Jx0A zGrE2E2c8XoOZ0iicwkK>*2YtSwG3F8FCh=>$6^6JVDPM>nQnV1^=q%Sa1lriHEm zinx-+!zjYsx`lt_KligpxfR7@XoQGP$G`1vG01uP<_J)mL24~~fZAWA_L8)FFbzOT zQkhLyNktO)1OE(3vuZV|Y^S`U{099{HX4y|csISuZ?wrlZn~RDcqR$woC(5H&J)%- zR3ro!5}X&aMD#|9+aLLhY(#=r!7YcOrrTuPv7)1)1(0VEdC6HoKKy*aCy_+b0SB2^ zwnUVkCT(KIr95$c)*d?ArAso*c`{oz;2e;-3uO3OYLe0yuaQ`Ez9s<>u68wEt}#Xn zkTt+qebRt=BQZzx2IhZ&IS>niffaO@2`OcBw})hwK~vfrMufZVIB5%#H<9EweL!;c zg+ktO4J*+|66!V#Jw37u{PHiZ#8z_zPk7w+ftC)Pb1ZiZTadb$q~`VosZ;+c_3&C&v2f)7rxOA3@#^8+)zACGbdvdo;=IAIH zY~HkP1KG_du_5Pz*x45gtDK4M+yN4bayYtil`d#r&LKr?F|5B8sN6~_FAoKk2hmK{@ zg++7_)^t;&JDbxuz1|d-({1GD?SF!sV=n#mAdv=SJx_=S7|8Pf@P)&xYCPP${N|i*pd&a3s zDPF892gfzM>g0qA#EDI|Ootv&JEnl@-K4s^0#xrI)sr~d8~092%1%q7_DZ7mN}?k? zBzHPAqmz^h`=@R%2*s-~-QM68f$hDAl$X?j^0y-eZrI8dfd@@;MUX3yyDUU6k<_4} z8_;G3O=mIiT+uihWEYX_MH4}`DuxC#w_noX>Ue`RL(=K*?iTpWI#Mx7>NUVSy}759G~f)NH>txbqSj^ZYr9z zVhr4ac2-wBS@;6Ri};FUqbwyc=5_u1#N2eQ_mbFOB=!`D$#kx`nJy_sX;V|xD+OYt z3Glh^H5;_-HfwwaKY zHchAyEo(t5SWpXyD5sOo^>RaW?0NiS+0^AiV{owwU*gkj)wIl=cLWIFsz3#h;=q)#Pso z*q!5#5V&wYfIk7Sp5qcVO4G_e$F=I`A&>V^?n!9eM~U_<(S88hg;?SmWym-gxdoT= zucsWQy->!@XN?^{t3xY${Fza|5fmOHg(q(Vg`VTwnSF``W$B;U2BtZ`B_QVbQ-FM& zkP8+7@_InZcXkjNV=$W`apT6lhUPJWG5Qs+1YSwtk8TI>BLJ>{KpGlq1;jj{7c9iJ zT@81xyCO={N!m8HNoEdpO^>0C0mGe!#s*MbMXG}qg6fP4o_paW0=FU=x++6%xHnRx zCORC?GU-U-Aytf>42mT#L!jnrV*auWmJR>$$X3~(-Q;BiDHiqoEKuqWrL9?$jFYz z!N_M|q|Mg0g13I6Wvt%K*9rjSJDYyM%)hK*K{S~-G=Fr5N?Y0PG-~_WPx}wB?2^l zpL}T})JNcsC}Y~J+hw2ZQ;%fP^{R7}#`$tTLt3$?NcW|uKzAeP>XW|yNC|fXm^Kp9 zDoIH0<)sB%y>g__2BYhVbMJcKG_3{yD;UMho43;=N%D?s2roF-M!|N{&8cEQbDZ2D z8EpvHp~3m#WIn_oL2*Bsf0)Hu78_VR&muYv?8vxudpz5mB)Ggf2P|zMOOI^=OTX6% z@G_il(OtFnebCLQP_k#4* zZ*fzq7{Zjh>QcIU`*9aZe$3)a7T-z^W&}6P<1w=@vfK#D&ye!CZJ->P2+GSSXq;6m zLs}gWB<)QR-=^4cY!9S{R-}sCn)A*3i_^rv(nKr`wvLKyo5mHMejqWM1$~<06uN2U zNNV;l1uqN!Qmzqt8z9B+I~!<^Wp*Rkd-YAQcg-YsJkj4PqKhc&Sgrl`B;RPnTFs@_ z5jJ;gZUzL7t-I0WXJtpe1Nr^Z|gzCD?NirKd-y)$?qNmI3%V0$08R z;N!1=EPF$iNlkjCgb@Cx@*dsWoYIoPQ8EaSep3;>2_#`7`i6|Wsf>rg9NM^KmfHst zou?kOLT&@a=Si{d15o_!N*U4ygFy=TW{o+jw6451w{PD=f_s5*-x99LRW{r?5^kOn zwM@Gp?KD%w{yAeiM;CyS1*yFwfD`A#p3|Ifk zZeVM+nY3>GFK8{j+TEJL{J8?R>p?(7SxPe8nEgF6uEF*a$#ni0WLANUG}vIRfJXEJ z4WHk8oC9ARn^@~K0$F=uSBw~IPk~Mu`Djl=ZQ4R&w~*NLMw_lq6Z6x=KV_R(+HLy( zBkHzrD~Wda1VmrcEu1DPMvK$9JdCQe8-zri80{CTxaVwC3gHIx%dCA9Ywt$w9>-%e zophwAN)wlX(@=CcAZ(Q`do~bb8!_Jb6d2{#2&+3VLJA{#KB}KrT3sPENMeBP1Zep= z0Dc94!Ne5F#Dze~udc=oIW@cjnb<*$l0CqfdM(Zjz(g2K6s55(b0y408b;l)_JDEi zz#O9v*oYgQI|=s57Xa&6FYuB{st5t86%zPY0mQyGshCk2Ve~-^0sewj`3eCRe+hsR z0My^Z#m6?%u$h`8=WCm`p%raI=Tct{lS=DiU}L~ltIJ(3f3_t;>_340DzWeX2H5+7 zjZHRQ=SmZu!Q8(jKxLJ$$9(l)WBuO$L;b_96ISk+78WlNEjz-tcqhu~fe{PDWm0f4 zr_Fj6y1We1!6h@Snf2h*pEO;_A{77+|Vb^y|qshk1B7 z#5E19bev|l#U&B)chb+`Z|@YwP2fEUaCg@cNm*&4D;5(Rlct5{!{bNepE3~Q5l~BF z5iO2lO2kHESFf{ryRIlKxiT%SleouTTe^YXwV0WTlXt$!cw zEuxM84QTf`Xy*Z~cTOM{4P%#8y&ENKYr)#v1Uo>mEdXmTBk3+k43^8xXHGLsu+;#2 zhhQsz2UwRIG*}J5uq4a-E`j`cpM`hK1kU5Y;Vm31x*a&)ZiG39MSXZF!U1^U2>+9v>HoFSrKjRF=zU@@CHDKk zRyVD70Ndva zd>#_kE@GYe7qE&Qtogul2cByhGe9LL$N9H8Qf>uX9}sF2p{@p$8g{oOgyN_>SD#ax zQ1bxwA)&@whvE9I1NA7NkaoB>ue1tE!Wc3RL$WVCBW4M(b`$FayaG~x_B3Io%MtS! zurPoR0;P}nIj`;o(nmzvL!?naQe*gWh2)tEUWWWA|0T-o5KwM%P*%#ed1rzbC9YW? z6DBhWFzW!LCPFPt#)6vRh<>6D8VXH{_$S=t8?Noq>}BDo-H_P+jvo)f&IImP#63XVV`eBZiiN;E z4zCRc@RyrIvptT%t^oR)K${x_=n@HJGdMUHC@&aj2!OsJ(3B-)7i8XpDrUSc}>^{T$#N>XrPmF2-Sr{Bq+I@nQ1s7r?@h!nN5Ny~?6+Ul8k3U4P!W=qpG@ey| z0r0*f-k@aQT?0Io=llfXaR45iIa>3|fp`(Xz9-lx1ltHOc@6hxJS=@8@G$jgM>LEG z1`VZjDG+}k;uXz+xCe;pKKp5aD}tCx@i@c(0dgN9{~%=QEMXlZ*PN}#*e&)>xgJg@ zxb6G_T>VIhl~F(x0irz}rJQj6jtxY?Fbtd-DKKmH6R9!JdZ9?!#r*S@-IYQ)f~vF zvke8qZCW6guyEXY;daS)*h>CNs7XfvYN&*=yM(@~1naKrzl&RWlc4N>Bj6cF0`MvT z%9ZzOxqW0@gn#vgJped4X}K}c|1<{k0(d5WC**QM&H|)bVHsfIU1N1+xdbJ|`h!@f zwE)%{2aAytkejnDF_3KIP%>%ir@O$`pM-jfP#;LqF*^c-BjfiG2B3Er@gQIh5T^Pl z!1TURuZ|2>iQh+fU>uXD8>&A5oWF?E>S*AM0gf`B7;@@snRy0fYfEYWdVg=WwJWyF z^}X@`wFI9FaAg@VI_O~Fwndy6JQLv85xgP|;QJhK25JTh(hDU!+q5@wfH#?VU0MQf z*c>fU<{I-bYcAmSCcyZW+5T5Bs;3b06+&JENY#EOEel}LK#AjS7oe^u)U;NB`pDsq zcdz|(W0<<80L^OLgS7{D;NOf{GYsS52GTjTHRv?CNzWQy@Al4`K%~DxWh$wxBb6+N z3U9WXvQ9*UlPmCVrt~jBQ>KwfZW|E!Cx|HN^X7cK^aJ7Ix?wahrxWvcVm<>*Wt8&b zd;GLZjOm$!qX0UCp!c=~=;r`cbC3x&{C(TUmaw^O|AK3e5Ns#_mII9o*~c>Kf~Pa4XFv&@Ti!6+jpDN5-)U$gwxDK_F~w%ya4M z;K~KmErb%C0JREGXZCLy3g6z8hwD``hf1`6gP6|1Vg9fO05^|t7j*{QPk=kVKZj`{ zAt>>{_5s4RTUG#KJ|PAk2Z(mJz{?B>aV{EnF;H+o80Q4>$;(Ml!J)7oaJLd}58?U) zu6~_`CN@SBuZ@VmBBENh62hAl6)I2GWuTLTaI8d*Q4BWU7E!T}+xMV$8>zK93DgG6 z6V^XKD@oC+Lldef2-=51g3z|1*++;Scn+Wo2>SNP0KEsGh))6R;R^PWsHX|hmq>!U zC`kflh2OEra3CiEf*Mijusd%ji5D_Jq84kv98r70@fN~4T^9`J>A)_)x`SBTvw-yr zuw(~r1=fG#1sezm@xs&QbhpFCy|8X9IZ*jx4&#`F=kzd^5<$pW#cj4{$hJ7XoiF z@n)U_JcgI*u3d$uuK;R(wY|gDquXrsxvsC*09^#oB?K+V2Ix_@1 z{t`^Z#elk(P`~5=YBQkJRHP*DwjXu3g&D0;$KjA-0>36`Xg4QBN)H4boL9iXxs)V6 z%?F9Q?tuTHzwO|x10siZARhCa*f5;aqW)?CEhErXeF3xsK%J>b=_wsgAPmHYfLx7y z4S<#tsIVV^(qI_3(Ln3yq;zymLKk1a%sUyz@_hvThM<=MRL;DQ5LxsTL7i#R5U4W} z8LZV6i1!n5e*q9b2cpzF>w)+Z5QFAS1K6Gklndww2z~QE0L@g!?gTGc;T*V4(+KA%hhR95 zT38CK)x>)LQegcTSW+#t1Xd=nd=oVgkmiV=Z`T}I3j%9MpmqcZG+pejS!aR9Y`nGM zPEHvXg_9`wX4JZ+j_loEJ6(lf0r*tc61QuNLIAFsn$flogxLp}_=$Bky|?j@UO%uN_@cfUjJ$P(%q#un6p6h9)j`GaZX94*XA%~0wRo8!x4V8&!q>J}J8oxJCpN2>HX`;@q0_u7P zl}}xnsOHN$9$51a2JR-{K119~E(dNSBv?`0o`ju>iR*c!-UICUi-Eiq$Qy~=y&A}c z4sx+_PR5TW*B2f)M)v^gSz;yE0P8Mb#g9?OLxJOi;Ccqt7l3_^u;q1tZ4Dn&{Gb|e za9F3&994UP`8+Y7ngGn}faz1wj5w8EBd>Akb>73gbbn#v)4&YDn0SHc?@t2y4xmc| zQ~B|kqCrW(unk?9?Cdx*vk~An5w7kE!2Joh&a@5{9MdoW&TDE0z`4feVSsy)aD%P{ zTz|O7q_L^s9(K5MHn#w?9o=!8+8OYfZ6?(J2sIB-ausGM z+`NWcAu)g3h__@i@Q#KfZy<()73CGufZdw`A~$b-_LhUa?S!ko9&ir>yy*(E*OZ9fa#R6>$H2z#bB9v0WEY-X-_-XAq{RV{yuC?Id8vbO7!KpzPS6 z041W#XcE*X0jSF3;roNzOT9qm6_Rz#^Xr6W!Ep_>y2Mmy5j zEBJjdV^RO-wdp-583_uoW$bmHu3%I2UOOh6Rt>M%lH8J^8o%P!T%um zMF7`x?^B7-w>!+H7ho?J(B3Zr^A=&Ay@l-sjBM{Z)D}@b{b9D(#_%-qI{>^*fF1MM zNO-m7AihNc@Y#^KMk*AvjE6b8Ie!7>9m0&gmCXT+Zq5@DiEod9l}@;@j(n<)Dj^on zhYJ*%G6Aubz=_GbL_2u_TLrWn>9yfo5k%qXK(_Qtna^}Ms9tvtIF0|*<-8YYy+>NF z-ws*>a7@q+<52W`oD!r@m;?tQ>b4+z5MIIH(YOjcW+H2@seJ(OF#&G70|0XXfEN4oq)fZ+twG6{m|Jt4$oN9ld=f5U_@9zhJ$t!L1HBUB)xn@+)2bixK zEWn8U3!vW+^u`AOx*DL;h*h9WC8Ik1P(a;DXF(MIBWUbFfF1y-G-VYi)9C?JDZ@l! zb=JHDV%JgEft^AR`>@O`_Z}+Fj=<4ga@1%AI4W4Bbt)5j7|hu7&!wJVLc*MsNpZEx z-VXS03IFgzfd9;a=Vc1jD?WFkdE)9Szveh6{f;n09|6pDtK+smyp0ivXP&#dc*g_p zd*W3-2D}~dcsv(o^M{U!m6ywPB|(P_)KQ$d6&GoMXqHLiv7cpCq^0QZ{`&5jIu z_Yzc86Nlq^D7gHAT+UhvF4NXPk*y;O$A~7!kcCP`z z#I^4AN`~Q44NHU|wH^;P85811f<3hsU@sbAdDP-jmVmRf{CdM!Xrp*^-?aeSPp}0~ z04((hWSm2EV3K;)g1aaYt;Wes@m1I=A$;S&cco_|wE+0jdY?cGlLk$r6bM&MJQ1@PhP zkZP0rcK}A~*eTVO6C9(;gyGd$AI$HBx%)Z5lme#yJ`7)BA5#0y*`pHxol{<3$!9Nu z;kov^^vs~4O%TN&Bz4J)AT{bKBxs?vwxuav_5^I7Sm{B4nFz_5NQMEs8Nh!MxW{Gy zzXD)dapFY)Zj^vR4Zto2tTV7M zuj3`z&~c$5R-Sm?K-fla0`_CTO0nV~GWsE4Jz`A+>60-brxNm&w*YzeMpMS-NoVvU z9&-BUSwbQdc<>)h=_6H+HI2k8-UIR6XAS9pEX5Qj{a{Solg4fHL==q)kGmGubP~*e z9|Tu`pwlmS{W88^f=`Ex9^qlr3<7>iz;6I3dn6kNL1zKb(fYcn4dgwEdYO%>^KK*&Y&G)(NDGlfD7* zT`w9UqDw4DL~-sgLpt=X-9orsgzK}}md+Q_`$Zp^UpgKzpI`_!k6@4N1=u2hxq|sG z5#>A7(hH~v@W&v*c!}7P#<3SnaeM%h^GWibKY--2mke>xc@`iJ+i_;dfX=sDiP3Ez zFp{^}GFXrLB1*@ZPX<8nHJk{y5uwq3AdCir+wj$dS;IoQ0~IQN3}`r!*;@5*pL{e0 z1|d9K3kbgQ7l4;8gUmA_vbKh_X5`)Xsrp}hK2N$DO&t7V4x+YrxZ}2JHwoJ zH-VA6$;k4?U}WD8V^DIp%t6_^C=rzeTXF0z`OXh~CPBgTGRFq4Aa?`NwHTcDkoq-E zK>eRP@g4;qeC8uUBFdh{#yH_SXG{%dI88#=;gTB$Dkf?L=%(VOp~)Y5Y~f4 zzU&Afk8zNxW#T#eJE$v&+WbhM&UH|~!(<3>9mLNEwYP}-y}R4X!*qLy6xNZ#dQfn7 z71z)j`twKn0#TEWGCZ0_dL`G0-0& z)K7#u=QTHUo$(f8Ckyp11TE~~1~`+*k&gc)|6x#hlvFM_8dN5OiX#=;sfl3y6UxE& z{lS$?9}JtvNa!~bdeJ0AyDylKXO=m_cZ@z`&$t_N?s3B3l?M33UN_o7>nadlSytGQ zauL(VrA;vLRubr}mH_H+0?`x*ST_^cH((coKgcpM*UT}H|0K~5_R%^BB$#j#>uwKRoDHIL(fMmg^t{#}+U*UacQ}qQ zEu70Yn8EI}z?qZG`(REDT)nv;ZCFdVUkEoIaBjRm!*)c_7BqERGCl42OyswY`Q>&Yw?fj6PfsV?|&cW$8sD};E*CT?r`vo$Np(swK()>8DE3?w-Vou7UpuT?*WP8QZ8jn>r-nz8XxUA#gzaj}BR zZ3+G0BqKTN^vo=xosUik*K(v~Vb9~7(S*a`xxOPDf$wIM+>;JDIECt}luYC44I{mE zq!(^pV9^|!R8t!pS6qfflHH(6np%)W{c`A}_W&EQ`Zukr9aD`cAh_1ckbVz;ynqna zJDOhBz_N<5v1$*&W^MyV;p?Q!Nbo)u@3IiqOS*Q~fOz5rTyIfT$Lp#bKDt-)C&ot9 z)!M19?E3BPf%H`S`mto$9!MODa^SMQF?Mxlx3sIh-le|EsTQ#c*w1G7WM*V_WB>e$ z;L3<~cACr**}t-UWULy${S6%x?j)n08H|mri?LGmjJ*oXz?7AjYPKWNd4D#(?JSdH znBQWCt7EUQ)dD%`Loc56dq-SL*@+rIvBoK=F$9+iv&~94E0am$L|eNx1Qe0@TE zk}7U!DW7fa41|eqPnaK32Qo)Af;z@Y3`)jSWQXR z>1cWd+hIMcdG4lzzipZxO`J_$pCk73EQEEBuEwe0YAM}Jt%-l{Sl^dq>5PAc8-vxoF&6G;1zWnv3Sm#WBsrT1n^MlFrDoQKK}iMvB(^pfwq^ zgf(9`!Bj%UILyV-z1@a3_F#LmL;Vy-df^TEmYjk8ODmB2A1ZgMXd0%f4&cfhvZHINj^Of!=G4^0#$8%I$)Rg3F2hK-hHb)fs_T+tLq?ii zt@hsPTwmj`+t@nil0F4P3kT+0kT+y---3Q6L-TS54=fm*UoxbqPkvFs;C>|qgNyTv z2Il1D_kk6Nnp5$FM<|4L+?hMDaOed?i}ME+7Yxcr6K}*{Bi7&)ER^!0Cj6=+78+sA zGdO8=w$O=*h5QwT9I-y_nAblCY*d98w&Sy4<+1#U zWi@#2jz1O*Q7LfLHIm6t&K79Bjm_QN;39mFT|?%kQra?cjeA(*Ub_ZuC_jY^m3-PJ zayf`BVwL4~l|hx`y(Ab?Vy6bqR=z34-Wqh!V%B^JHEnI~ZH}iV)}97Jf)(U0Ftdb^ zk0{8zy6S4==7u5QX#^~2ha(!~c2jm@4&KY!kEz;x9prCRUCOF!Q8mA!rmi~1T-5lp_%sB}LnVRE)p-+`e!<+Lo#E-WrA$s3rTGk92G zNxz~Y!wT`Y2RPBJXY@?Sa?N#H%Ndql8NM|vy^Wj_Fs^l7Q&T$nUTYscw?A!PKbIus zQWL?6pJXa$;K1CRya6TogNGLs4H*o+9>hl>YaicCDCxnLDDyK|;pe6ZbFuv*ReY5y zzDX57r;2;kyiM)gZ&*Pe;5L@V+>G23kK(#tYNg((19;ohS?cH+0$Kvdhjq)?&8{cBaL_);6t{0H;ags)wN{U(EOs3oV? zfe=zF;V+osYdNx!GQX;>wgxMT62;;0lRtq!q#oT&vJ|&c#IqHe)d4%=R?Tsp5EWZA zi&%#>^Hl?5ZSi+u-Kj}>(BW_id-`ABniOpMkdj7Vu$NlCm^&7~jJ%6uE^9KN%1zR#hLpoNZv*0IW ziFSgR#Uf%Ienj7bfgGf*vD1oJZ&@WZB_&10#Ra*@=a5%kT2mu;<@n%vDY^N$EB^Pg z)GK$yK&uqT$&zK>K4eAfYSoj=)t&4--5G`5bFia=haoJT5>x7}Yv&1YvIE)$ zxyYHUA3gH2BLVJS4>D!9vDMpE(>pEKE{73gN~^gCO@;9o-qO@6LE_ud6|s>e<78xu zQX=(MOC*xP+}VTyCae{jY?kRuam>d??$-T93hp7&Y-|n5=IWpoabZ!xkRmFJ!9$7$ zOEea;x8 z_72wmA1m%vwQJ*R>(L%yDf^KwCkSn9@bP#&k~+q71m&;&2rE9SYIA=jThUeT#}ep} z7x;a|D$e0xw$0>~SZz3UH)Q=PorFDfd!;?=t}+5E69#epEBH2i zcy_YLNfx=u;vdOkU$QumEUs6K3~`Ke?jz_zvdd4Q(RU3C(p409}JW2|y zH#-yF%UtQARi;A;wvQ3(%!6&zfX(e$W9!`Dsa!C4c+Nl=LGD~m0_-zUC6HVh%wcU1@S_#HrU1MwUGQ4y%v|q_ThU8lHx8poW>f}b~^tVfe&ss$rKJLD#p^SX{28fvF`HHQ0=6P4I~H@vDSINsKLdIiVE1MICjA|4@-G0 zve_6sgrX?HKDA(APA-fQmzgl2Hu<4=tD&w2Fl!~LxBY(~T!|*&ZCzMtiiJ|@>hWf4TB36gy zx(0}ima}~~3>S|<*H!L7y27@ISeqRSOj=+4%VO1~m`9Ub?R-1x)H zV>o61qbFFFX=K{dD)Clx7pFq_MZ^9XRUM1rGy@Z?9tRrr0xEJh+Lii*_ax0cHi(x> zNFJx{K;4f}cMXoy7COgiZcUJQRchd=md{&@zkDLw`|pl=&L4>vT5ym+89?e z=MAN+u>Nw4j_}R>tb&T#STz>(v9Xc)6Jb$TjE?1u9Lb%H?V6nfruU#p!unbZ++6^i z)+ICy&0QEz-sGB;<@g(&dj3-dn$hAZ77=TZceT^@H(Ei<8xJ4Jh=-aIK2&NSi;m1z zKVTdc!1}`ZXef*?>q1tnwOnz`Db_tVVBKkolZmv}qE&0rzO^{6wK%@DII*=jxwR;3 zE#6anozY1yXW-)qNK;rmqf_lme6APR&aDb>EQVt3IKX{`u@~;0uHV|r&MV66imQjb zCyiq#8Ha%LUd)2}#h43Upm)MJn#D8iy*_z%wzyo z$%OcM_%3AeHQ0b?u!*fH89JtNg7Fj@T)jDPkILgjVP(Li*zDa3NZZ8W$FmV|b^(i$ z#gb&PEm>SazCzXuwtYV`#^rG^3^~WVKNrcX`X~cbWYj|&rNdnwPF~PTRJ9hf+lWY* zKfHk-W_p;FA;dgYW;(?d`0#svxDX%W)C!J@Z1ZcTRFHwJT((<41f5ag2UfT-+iXB76#${lHkoXk=~hd`|Ic z=o0|_6Te@M?|WZhe&>q|qPX$kA-?~a-!BdR-t2Uonc?eS`1O+DuT_#Vg?tje{gvN7 z$ZvbPoT%VrH9mNxKsHXn$G`F8`vX2!Jm^nP#izgX(}(b>eRsLQ5J+x*f8Cw*-V!|n zRsLX=$590b0OLw)jS4yLdyZi%8=q(4^FR6d8hl<{IW|_o7y`2adaJ zKT}$YDw(r>4nNDnX9HmYXS-jTAwm4^CVtly-wmsvt=zzuH%pFBQvp%nw-IWmZcCHv zVt`c`u#tGS3pNsMQEe`uwx&g_PR-*g0=ON%2=7GpN}he%K}v2E3vdyllw*b{H3=hS z0Q_sWur05lq8Z-gujvVHYSP%MzXLVrvF67%$!xcZ47&;0f9##J;9 zd5xL&2rAx+imBolDLj{4yv%TWA79^wum5Tru^Ol7Z$bZg{2_IocjIe%!{fn1iy24y zD8|_W0M5n6cZ##|b!!58B|eE*dpe${!X7Sw}Z#ez~dVZk9IXMxSw<7I&63`4KlFlb-g)-m6hf4qLVis z?-ICX?f_tTvZ|CBPzlU9fqCgkIP^wV2UNS0)uza5L&wzBj;x$eflIV5#hUB@EDRiT zrV-~Z;>?gZcI5;WaU<77mm31lOLMM4hfrMLE+pMWQw$)1j&>$mm3M`#;_?qggfv^M! z5o;tM;woWa-i$9I)Vshd;Y30jZZP-pO<5!8M0oIdb3BI`b)IWas0#x zEPN&xYl5jcn-$^D?2vmwalU1xhnu#rMvacu zP8eZb9}0)Kwzb5O&QgkOJZD;0bvg5Vd&IyoTfYtyaTz`k)*G6>>q9ZZ1RQUO|9I_H zGVz+?q6XrFgK9S;ob9eqg(`UNAC>y?M7fu1n9h5;TCxi zSmBHLVW$5>*Iu{{AFkksm*T^rRizUuvdxdBWtowo!^gPx-HsX$vBpT$IIpy^x0kUH)_532$|)ne7}0U^ad^L5Mp%px_7k$N zbi{=i`w6=7vk2K~I=-}|9C3wcLKOyC#CpDiTy-t0LpapO0$6-}jT>qD6UM}&QWksf z(|%axf9=)FxT>m-`Q~!~-5BU3^?67iW3RP9XaWb@px8J>^+5XuN4>uX?e{ppKT3U{ z>HXeZU>W)TdRH0w{%_P;$y&$ATBh+9S3e*+ohbDm0%MhI{B3YTo7Eu9J@})rh9T)H zZklk;!uwt@Qr|6UkGmzQF?Gc2Mr_1S(4_V1Fy&i}uGJtl0!O@^o9h}N^8@}6vA)JR zfO3MHGgnEyAEX|#GBB$-UV7;B?C!a?m)7IM<0>UKqASI7EM8>s5{uJOUwoqm=>vM~ z@uz+DV>-HPNZ7S3UOx$3gM?8Rb$ji2xz1 zRCat1m$K6t3H^*i+K6-8h@ox7ur?yrMljw7%V;Ts2YitOI2DTH&pc7~H_v5^N7CT# zVi^(V1b^6pYYvN^@uI>$ikpdH&BS@l#HG!|?TWLJ$!_q28iy)UqfYbY^qDC;n@D`u zjVI~^7Q*^LL->cEUgz{6)R%Tv_E1+F>_Nzz1NI#BB)dFFN7_)z3|Jkv*`W_%P_;tkU-gH7)MJ=7aut9p_(4y1m zk^%(uD|=JMx**!O)ZVQ3P%rlkM&>g{30KQFaDDxS=F^4npg$qCQbwx$0>8T$zY`Xx zlp6DKv(G`6o~8 zgN466g};TYOXev?WS|0;PzJ->T^|;nDd3o(DpX-ODvI-2T*!iFiDJ)c6sbSU@ZEg3 zVey?2l0l`fKosl2#s1WY)g(oKk5;`dWyCS&9USKPOkl53Az)xKU`y5Yg?IUf(_{QT zV(r6~T>3k?y{N1P3+=IHM66*=OE9ZT%Vi)&{h2T!g~drKpmEnr_?LN}=!$VH z!k@RWO2*erQ2kgkqQtgPL9#K%@zW58ux?CD!f(|3Da|Ai?gh}Op@9)Wprj5LU)NM{ z)$dXq>f0AKZNd6=lRFG)=>z?fboj2^YmMD9&gjlF-q++RIMYMgjUU1XjN*w{A9pMq zK19Z5)1DUA|1>kyBzj@g+T*CjOXS_rFHWyNn|)ww4_f;Pdcwb3@QaDz>w*`SFjl5% z7-z3x2ie(@@J~porKhX9Fxb+{EgOGD~v5iOlv zTwRKbnxm;=u!)zBTigFApaNdzJa6NCjn>l*@2hj6vYV+kX2N&0jdePs4(&?plrBYI z1`+F?W4o61wh#3W1I_KIAF+;ZYg55*`=W2clPn9l8nG8pJ1_itJ34#^K2E*T>jerv zK)VAw%ANRn5?nt$9M=zE7UB;Pt6Oqz-@XMn*)K(WIet3{G2RiYZ&T+sD<&%g@u3Nk ze-j^Ihl1htneL}7<7#Mv{*@#Bb9Pm@%~8f=bsBU8Ch9A)^)u2DS3e{#f9O!$m+H%) zSsA^Al=2Wo#LDF?&|y(l2VuRX+2ANQcb|Hs>wxQ>UFx49*&ypQaw zZ;Cqs0=q}hJpcqoLP&&e=DBgg_A``T@T?hCZ_-uSTUvzfR#5j`JGnkN2Ab%A?ssrh=zaOguU_F zFxdwti?@@-JIP{Kve=U>IQ2yM3`A)Z)a1}2vV*UAv3f9%uIU?1=Yhu$=!rd$Uw9Zk z9$Jcfe@ZV5;+HtG&?g1z;8azE3)4CRVKuCF5vuixL6MFttB4Jb#YPra=GK*!kJK={ zaPC|(14&#qXo9+5vXghp*j5=f7OccO;>)k_<@OYWn5U4jUcKyi%=e_iZe{-xj_%s( z%JRPDrSy&Nl7BnK{u|>eYwW-5mVfDF|3xzD1d$@=Cq~c-MlA+eekOn0J(z3I>sTbnCycd5^ICU#@1d)Gq7B@L^IRSzni zST?S1oVtoNl{agXz#VN0pVHFW!)uZFHQxAb{_ z>G-8%Gb_KO5SX<7;8Bpy?$8qpseh|ecq~S|iogni&OJ7YReyMCbr~aQd-cLk_Gp?a zd--%41^UnE?0fNBVZG_J##H?YHMpz%Bi8Ry2P0|{IV+*GI&Slfj-+&+&*d>2!rwB(MG$Eoo4A{3XmCp^`>mnUqDhAvNBktT`( z1P%48Y)IEEHlz+undyc&TYmT4uL0xLyg^f;A)N8$Oc>*i(H3e~X($WpeqE!R*FlxA zC+O4?53&#zZ*r<3srQEs@aej5Bi8CO?ZA+lyvpkSBdcMI#H!0iAVL}Xt_Fjo98)v( z4w=>gw;^IpjxFonKH73@m9*PIImT17%^+JFI4n?yv5-(#IUHy}9vL z_v}K}O7HkJ-7R+gMB`V*nLd#$=v2fg?AfW%4uGw1F8BA`Pf3p4mDiD_qd@%Y+9w2K z2Lo|aGwArj(rT=pkbNDR8r}z^k*5+cB5G<)pT9Z`(^0}s@Z&~UhCf0+;U87S%z%m* z2u=_H&nTW$XB5w&>V8%|AgdPR0g&vV+AzMf1CU3?vU>eyRQ^ehi0^3T`ng4OcHoQD zOTAXQ6i<+O71~ZS{Q|6UKLhjDMx$bed((7h@+!9K`Uz~L#<);GsD z?@j$8Thlr{ub^T?c^xhf=zf`vKMISch@|VRmtOnDKQHNlK^Nbi8+ScFWZevr5vzyZ zPy~;}g*+{hM<(g0AUd%)js-U);$adH7Im3V9@@b2C_?`@oX_gFvIxTlE}`eSWE33M z&Rw!&FdToCGRbVp-CI>{wOnxzRh{)jmkq{}89vPuEbidH3Dx+U#D5dyJ=dY#T@A+( zF*?e-Zl)`k$p(H$^^n!vd!XQKc7lcKEKg0EiAXbXbTctS335}sAhi*kU`WwmR{4n{ z)=PF94sxL=JYJp$_Rc6mQk}|i0Vmu?VG;QQ?FvtlU&7_UPqU5hRi024shWnb|CBnu&sDf;aRr_EzAq0LRI- zl~rn+)mQx|+kc)Mp6@x8wS#c=4VG-&4A%EoCSmzR)382D{V>i(&V2j&*Zm!)#H-FFqS6 z8%C3f@VE3DDQ|n+jit{p{Oj}BQr?^K&0xboO3+l(JUZBwqSHaCFO660F-qPC3~-$qPqBc`;cn4pqby7B{e%#$q}P+PC6k;)b7UY6V=Gybe<-Jkz6Q`owT{Jr3!b;6Bv) z(No~|R*X`tsETO~0-27~s^M8{?#vzeH~wjhOy z&RIF`Qk}DM3hd}`3c}WKL}G-kwZn3JJsTN0-*OXS7FTi$$#YnaSNjs+WAKsQz)z=p z=TP##mIJnnYn;dmHLv96fqHq{=_ejTwyw()@X zkjFOs0vh-W1AQBT{~6%$51jUV>zww;N@!hU{1qx{-J$MG0q zx1f#TY*{Z4!Be#{`x2+V%Q=)8VByj!z<;tY!~tIc!|ik$p&FPd^MDYsx&lJgfedcK z=oc1GZXBKLIqcDDS?9;x)J5Kw)rU%Jbj7U1K2U!j%VjQPJty5%r4?o4vU~UFZJ$y4 z#D;^)L4)x8E$&KWcf*|t<^*ll<`-wRE)Y*DSf}*<43`9qTvK z8I2-1B+$D9a!TTIue3yu=cz8A57A+l+B8;OT~SF>9UJg1SW=W7l5%jbaHc!aZ zQs$9#f|2$*c@Y*Jq)rb7cvf{krP~KW&vvQbNNV=s-}kz@$A6|}kx=iL(m0salZnwZ zTsDwqp~I_dkZ2b53jcl=V}(yGlQ)m^&6PBM!Y6H3UbNG2_Phy$d30+yt6IBtkPjB8 z05#ww6<KbHy$j0Ny zJfqCx@1$!HcVV-&QH~b|&M)xAk!N@7kK3{E)rZ3z!aJdxur-+Ab(uI?)#oTNGxOlc z=gMS3jf}DJm2Qx>EtxhT&sqJft6K%15x#b-UX^`E|lkx)J!|hP|}FV*Mi3 z*uO69 zpkN{v(&ZEys$+|s7LzoLtC+OhO)r@(cfVp9Pvi+KgtgI8odp#mV-pWifA+_r2rgF3 zp;v2#cgbekSIV7b;@@cXR@v&U@$3fZodn#v=Q)k7^=IY{DbgVfC4+MQ!GC2b=Q-y> zCe{-h{;~2Z=_Y&?WFpqn$9Q)3J-HqDU8<8WOw!z(*b`7w)Y)@)o@Qjvjacs?#nh;g zB{j^mQU8{taRN`MWIR;gYG@Mt{y+=TUKR1o5|Z93t(wD|20k!^zYmj(Bh{B9RIThZ zU;2dcfzHMWqp)a!$vW;RjkDVPW79QHVFF1XMQ{catr}JS68$gptnONDhtT4;Iu0$k zFu$bOvN~WHwo_^`B&WoQ$%$gh3+p4zZI-*v*h4aL@EKhln53ic7FG@JE0*F$5JdK# zoTM)}fIeqbVmLaoFezWaZGF0DipNzIRaVw!n?^@AGIQBF&dGP?#DF4q2ySZd#Llkj z(V~8w|Hs~U$461E@9%B`2_zu_lT9|DY=BHy%>sgASQY7B1&o?tMU)nLKmzu%6eC`- zp<)+M0Y&ACUdz}k7O-OP4Y7;udM#If&-0#{*(tkA;(EWoe=Z-gGjq;+-hSRw=A1L> z-r11_|EE9(>9$#m7Z?R+;w`;u3bB{dSoSmsQxti=V;3L@A9^ege;N)gEJIYx^PPYN z=!pTaIew<&wU@BGI&luQ?G!X*A1>Jk%X@S=Bn@fX#8UGyU#9JcNHF}^L$M78ao`AR zlCAFnXcYtau0;0xXcepJ1KFQDqO$Gi8BXN9g$RX8Tclj5b`H;AVyiz_M$?N;8(Kq6 zV+4`1oj;2vG+uu5@KdP({X8z^j`s9Es5;zvu-b334-Z|eb{=3vr8P)r)%m zDf?TIZ`Oza3Dt+0>*VHs|d}D(l`XJ!>1bE(&;7o?Cnr+5ALrSxPV?-<~9{*h- zN8zZhAkMQkCLd&J1di9?3JKso2&#kE*fWtj9H{}{uWF~6StATJy4Ln$R~Zh4dAHR6 zeyRV%QvbG6|7WHCFG~G?l=>f{p8USw@Yy3yUt*CUPcI+jh;}L_I6uDeGr}J#`Gcmg ziI9|-7fjIrC1wTld^3A+aG(YVI|Y2db)n|=N<=kF9yBhX7=IU$AUkWeudRP)KZog< zTlq>(y4}msVYiAoCfJl3UJUuSg#53B{I7-lZ-o4Bh5X-!{9I25HQZr}(=YXy$?a(z zXQAjbW>LnEvZ|gn=hsUeB0^Ij^8)Li*tY{CH@I|SO z?D!vN`g}RQ-tyqj@$D`***U&`aud$+?Wy4S;gTHRUh;simklsd15Dl9c<}RKU*lmA zANDgIFtX3_4VH&NIliHC6UgxmH?nh?y}x>hgSQ;t0Y*-k4@O*-i zEj*v39u&`~7&*f8Y4QM`&oEK}lj9qPewttzAvak$zQ4&0m_JZ%+T{4~c^y9%W9Rrz z4*Ig9V;U!)GzTZ-VPV}o9@By4b>XR~`IF~{qhTwnTX^=|a23j&jxwD`;EX$g4`Y$V zq%hVXPn?QH%i+q2fQSPkBu8I{(Gg&B|#H!;j^2xGb9{5f-G&lSVq3@1Zd{!(ab@XTrBn-uW)nV4pp zbaHr-taqV)v&(#0d5LZ{QaU1~eJ^$~V79qP=!ArhR)X0y=lFI)dQl=>R-^;G49PoL z$p^ZAkGC`?J{F!d4ZDGPAbFVkE9rYZ=NY<-0PhqOD|Rx$DFJ%n zbYvYgZ1&`)#wvIi7f6Q3H=aCC+&XG#H(ZsZY?aguX9l&y$}-zQ@6gE&IC2(829n-f zWR)3B6)K0@h}R-J8;j2Xc|C6P5`#~q+=7(cUTR$u9@pcso7+E)?5eQMNpAyS2O~r6 z06^*6k=QduX$9oKfe%V#6`DAVK3$IQE~G|MAyH5cELgFel?N;nupI>_4}*<+ky4oi zg8|_v{TyI-t4b8F+F@1u7fMtN9p6YVnKT(6U78f0ZPYf0gZv7(Tw^gWm^wJ*%j!Pd z43G1%B*juE(9E1T9=j8GvS^O49*{jokDu9q9T|v>=|GPJ=)UG6#xT#=4I^gEn>{-` zb2c5ea(^UNfQo6q*pE4q%cP+Fuu`XhUel8T{Hw}Asr;D zs#QV)eHCK-BqPxsrD)@8L8zAtiL1q-M9xNL6$;_B28%m`IjIIy(QKQ+e!^sQwi3?sr3p~5l3SvAR<9!K2@Q)AL7I%3+}D4wvXC|s=`%kh1H ztV$PV$*WRQ(igG?wW?R+l4LW2+bj zt#3iAo%PN%$M-9q^Loj|zv07JQjpcbN-%ma5J(BRTfry)8lEg z=aJf@%9(77I~>%^n>;vQ8-VOV|IGKMoN1{ z7?BGgKOiBb5=4Q9rY4!JVjE{xELDDrj?DN*+pr-U2(o&y_l@0|S#6BY_X^y1!>9_a zU5r0rOx%QJxn@|7HNjx6vZ*}^tz+>@Ds2Pq0^@Ll57;C0M5Aax=?`;vMqD>rd08GN zTP$;Yry}FPG3v7bd<}$2%E^t=15$je{sM+SR^x`xxM0r~J)i9{TIg^S`#U(j3TL+G z_!a}lO|7`X>_?%M+YdHfug?+1-nk;F`C6_wQ^!rc+z|s zfhI$N!sogQx12r-Uyj1VQ&UdDKl@?Ti7_>Fpmw+B_^wLB$b`x0R8CIl)GUS+3?0G9 zttcEmKsh)Xl!T5@(amlpz-|XvR^>F@-GRF$Zh5D*WOyB0u43^wh;kZHS72XlIKXY2F*LXOD9VYElM#l>%lL zCydqO86Tc7d(z1O>I@((GIMlGy&MrZ?B)lh_dw~$VaD`^QlZrsK+T+M)qZ36vlrlV zIb8yGLGcTho@BMyo!7Qr%rDviwnbAnO7kp8=$1R%=TLG#^d# zCtyxz-hBF;srhHg%weXZ(lb3!*03ru8Hkr0Y6kSpFzRh$uB>My?Cf*0(N^wnb()yu z?PR|TMArNts|sc)t?4@%xN#oJIZ(*1U_jm-BDaydzPQVmyMDMU#+{Z>SQC$H3(rxf zsL^@V9F<{{Y5@0C?+(MWrlHY=t4^BSG-uj8G`906^58hBwgz}#fLA4OD^y1Do>(dc zHC9fGm^xk#=r%*U$HL(11>EOXHl1evN79!_>ViIOOew0T3>fI9Y{r`+>~s7GkQ}QA z^0#j2=_aTggHqv!MBVI z?R*d?v!Mbtg!0&IXMt)5avX?IP1TZ;Q;De_dG06A{nM$+U^a|Ztg0hs)8D~Tyl0}g zva?yltH>!*piQsY6Q+ZkMtN!*?tRo=wES%I@f3NSqhO89|Xrlp^imKc;%Wk(50X z`tW+5no~`x&HRgLR8OrjN_dsIknn*9{^I2ezQ+()EcfzV1uUp(R!?WE%Mx|LG)=aA zWC0h>RaNBSxadaGob7C2&I8rf6&Xk<81a@?G8|?t63P7nE(s0t;gm4QXyb{(OOv55sw2#-pc=@pl+6V46;DoLFsy*%POsTga}X(ZRV?P;^Vq zM!4t1*>mU4pEGYiHqXAxDQq`~*QS`M=_H_q;oeR1g%Ek+!G?Ogq-XaY!M7~;%R`TZ zip(})n%bi02=tE%@GdZ(Gy%pgz9!U&`QcgW4H9kB`3mybp+}EN^JmREnNy!VdLVf( zNe&(~dmieLLQi!wsKt;yK(gC*pNWmuc-&te!!o@R#zfatlL|(j^TRVHh>b~YtN=z6 z5@3B*m`g00=8c~MP<3Gj>Vpl+pKB4e^KuDPAKNYxop7E*w4e1xN3StA&%xhY{nj$=MH1NCWh~a z;_H316(@keu{0<9N>*h8y;QDwwkan;ZdAqL0&VMo7|f&qA1UBnWWtGRMaSLbE76m< zT7a@D8Pczn`@pWaEyNiwzO3DQ_YQN7PISuTMh=XDK{dI;72d(kmQm`g*^{OTmbt3Afx~W3P8C$WXXM%+_wW&Q)ShXV(Y9J(ay%0 zp{~9Hxr>SjR28-F1^XxVIQX*mh`RrUN8==`FyF}Fl$#SKhu!&6N_(di&gagScC4f^ zCf+ZJd6j*8^=)Xvdv&HgAZZ>P(QqB=Hx7VHj{ZR@)jzzWf4v20*XiL)+Q@6ZsYa$d!65YI{o(x}pRa5=MYY>UK}4a`%RCWu#5e z=J=0r#QKhvDN;By-;>N7T1zOtZvh`VdHhVlBbdZupGf8|Aj7qPGs8m<9-}hx{)+6x zb{B5837FTgsZ+@2ZdkvASxQ9VN#jp$s-|M2lGLVh{!~Wt-tyG8vhVy^eC#8SIm)H* z%w>UCU}di5xq^>bIclvcvd6FCbJs*%v3Ch<{n4oc=j}S!qfVp1lqYbhsAC5?75fH3 zULAM<^4sLOPvT5`);wu$Ct+m;L*@8M95~76F~cXQZ2?pZ4+^GOxb1}XnCOv24d<2C z5p@YUyGV|9Fsmoy8jc&s$O=m}Z9@{9j8zJJ1e>iE48$G~@H}imLmO-KyFZe&kf2}> zC#(i!HX_>yp%~|?R9}G0N!cm@K7rm%HT%;V$IA$M8doTJ!BInH9_A4#GXSGL@&Y&GBw53> zDe#y{Cfl$&!OV#F|3x>FrSGxN59l;9jNa*VGxs~)V8 z`=R98Mtsu3TTpbNCRNfX@R<}SbE6x9FQN_~RsltPBLLmq0DLjW<;tAZx57jLc06Dj zLh6u+C=jOK3h4Y<;mSUHvZZ1L>>vd+RY1|SW+{?@?=9eYhI2PengA379cdF98j!|K zKwJffm3xOucBrzca{k!LK7Efce`=06wz5yn5dg*~`3$F)8r(z1?WZ0F7s%8y&_-=j2R@pl^jdgCwZe$Mf|E2Z0@I?h6&f6G${mg+md zDclpOy%7+1Gp?+#zX(K^A!800H7q=M2tI%~d>CB3uDCFTs&3_2F?IViM$D7%0E+CuvxWG59*?r_#_{er z{2(Od`vTl_g@jY{@iT}&gZVRrKL_$>6n-!e1SHR4Lc+yNZRU>)a*0500OWiHc^83J zB!OHdkWT=Thol4One>n!0+Po70&*K6U*XTI{1HCi=AGj6XFzhjJQ99qYBGtwM+W-C z1^7mzf(#Y_vK@b166GNc;UAFPZ;kxj36#KD!AH?jcYz!R$cq)^0nFNmKQ5dD1@ag` zatk@~ClIKfKV$fl(EQ;7`CAuXfUCBV<=+`UV%$_eIlk|Ud;uO12FRa^@beLW629;$ zAH^3w=iQh5QNF;Bd-?)vRZ)K35jp&ZCpEwWXM6!}G(;j#kHLk<$>736W^myNGq}iS zWU>Q1WrpQK9esg2RjHjjBD*bru!Grf%mUsCGljhC$REYb#a(>?o|=J#%evx6q-f@y zc=Tf4Nx>z&`#XPB!Cy;#0WQx+!XKshk%I5??tT7z$e(Te`HVkb@JAKgh=Px+g3mDZ zX8t7fb{8Lo;yt{(mp`iD_|Cq-)2iTPPCW_5LwF~w*78mY*70r(e^kM*LcRc(&?Dhn z&RGv2QeEm@;D77oJ|3w;5^=`0K+?>cmnTE z;*X-pGaP&Y9=w1g9>RbN4`jfF$28y~2zYJ-?ghnJ>YiseAW5M)?qWOywkb4|_>dhq zOkENtIzlA~5}v;f7+a|07bbjxFYl0a9^Hm)em?>ieoz7zeo_J#!-e@-35s4Ly+-ag zMjpRkgG(Q|V2vzO1Kg5~y*^kbo|V_jI*S;gf2KRTklE(j$U%bu!hL=Y3?Vy6;%$on zbIW9EU^V;CUYP)&>IUv)$1G@WhFhut8Gr#cJA6K(+E$(s2dg$}v}%S}99dUQt%gy%6hK*SjhXRx4Nc0+Zf zaB8aF0(v7UP~y#on1(y{9rQ#6^b-ruM{W?&r1*#h!>zCmGy=nHnc^2xU%6pKm1@K- z06cTRfkdjf-A&A63t*V(0xWSQo1<2CP4O511OMbckUc0cl|Tqh+`8@yNR&wpJj(#) z7I$Aj;the(u}y+`t^v%g-M)ato7BMEyiGRF5MYTn1P&y{%;RFEB@ti=JE<`*Fo2f` zutc8Jz|98mc>*lqCpGZ72JkWgmI#y@_(B7iN96ee5`$6$UuppJ=p$c1B2j8!o^V8M zaC5RRAmNC>$<4jO0A3@&5|~nBzQF*#S%4)(rN(@d0nF37d;tkmsWGoNfHw%R1g+GV zxhIeU>H;iLD>df34d4d_Si%>9lRMJ=2JmA7EP*UF@FNEBQvxhOEj1e(4d7=5SfW~L z%$p40mjqbiTWVmQkV0*|CcqNrQUkwY0KYB367Et1zi9w}Aixs#QUkwd0DmID5&#LD zJQRFv0DmFCpUP88;Li-;?E?H&df=}O;2#C}yY#@{8^F^9Si&O|*(gsE|M1i#PZHoT zJTeNLB#9Ep5{aivq6Dn$fGcmIkSD3f*#R^09!Zocn85OY zBuZ7w4!9`rumE2`YL+MONy(Fnlt8vw5~ad;3cM(JQa?O-uS%X&2T$Hxk|*Bp$$MY& z#MwQ0TP06?JHg=Rk|;i$Nc=_;#a9!``au%KCo%Zr1JpW|R|(7v&ZsH@@k8Xy^5lq5 zB4@5EClC<-M9ztx9C1?QoaV~O3W%p7XMrmx5D-5_PO~RRoE16edUC{Lk#nIZN8A=U z(tOz@4Ftq@kt1!DBS#z$iT1Sp}K5`_g zI&#GQku%vPk1sHVDcM%1bA|_CrUH-}0St3I0LLo;sTcs5?*TYL0Z9D-z$qSpQx$+z z69Am)0f;LAsVx9l>;X7S0Z63*z&Rd(zbgQ#I{-M}18{)?kSYX#i#-6BC;+KR0Jz)( zaD@Vp3I%{`JOI}!0I631xXuG`y#kQx1%S04fEyKn)G`3v;sLl-0Z5esz?~j|yA*)b zECAf=0k}^ANCgAHLmq&K6@b(;06gvictQb4bpycD9)M>QfYdqwJm&#;UI9qu1Hcv! zz{?6i>L37K_W-=10Hi7c;2jTu>PrQrMgrhN4}j`F1*BpEV4DZvQ$<4RCjh?m0DPqY zq?!WYTMxi@3P5UWc242Z@kw%8gzta<{-=TeY2bev_@4&;r-A=z;C~wUp9cP?fxk!t zwO!-cWh;HP;rQ>q+I|HMox0@q?AA7Km_IKstFWjHYuY-@>{PS!kl`KLV&1SI*H>5@ z2)3{FcdV<^%G-~KhT2EV+K;O3SXb-sq?ff{+^wRzdGI`+94V;cDbmHxKapx3_m8;R_vk*)dj5)j*Seb9fW+yI~#<@);amw?Xzu)Ym$ zHt{Aq(k8Dy%P6HZn4O2r&CCo$aNj~>#WA6mQra(&1!>zs+FVoAFOdd9UolJ1RTAfm z#9uQBS>NEQ<>ncg5W^HH%O%ow0_NEOp`AW9-Fz!F@~EzU_=~^uj?dS%rcj7|XAt|I zIP~@pEX>LB#r-%B&>%a8dK*Q>{Q!vi+}eQH?^izLL<;lj+ln%O<8yW-AIgli&x7XL zw^(yYT;yb!AOwXq=jnm`HA5GNuV@~6Y1lV{USg!MZe>^9F!aN4pBuV;5KdI}tb9<< z%5SONQcy1k^|SNzY!7vnLh5a8)ESbP7XKZm7uw=Z8#2=-Q)W7YnI(C8Ta%gPrYcm5 zTNNfVzOI88cIp$$ArCp3@~{hdI0r5U9+uD?mzzAO6!PG!S~K*ca3A_9YSQ-HSUXZ~ zmnr40p#1kdy}c>@WhP~nVp9G%+-J?ux5B6z&0!ztH-}?+q@R~5{V?dCo2Tb{Bv&aW zeM52s&a}y@+&*TLg``e69#W~MfUFf{%31|jTbieL@Tf+mn5-G9G2l$r%$j4k3wg6E zuaF!TX3AkVaCm;6USuloS{N^Sp2WaNr~JCQ`t16w$WHnF;HbU7$Y1WF5(8$4vR`4n8l;(@RWEUQIqPP<)n}e3lt}mgYx7`39`Xr>Wx$lJW^> z43f`arhIDP^P)TngxY`dpn%|ky4gR4)%u`cAQOQu}y3a&2A(|0zxx{h33qPV)+mcH8L zs*+q?0rb!(p9)%2EI$<-~2tD9}5(<@vD}i8#(EgDJLVr$zyqXchh7K zQ8|z1MITZ6dDzTCu-O{+p(%NUT7m-=#|KkQk3vn`b5$r3$I>6kTi9{V82pVwDmp=` z;`bo8Ju>BXG`M{-Pp>k$y`S7}RIEN>rdzB&V6nO{S>0&!emiSD$y(1$S*rzWoAUJC zOx7MFYtJawo;K4h)*iQ58%EZiNvgAIGFF`_W0S$ycX@hmlQEp+F(UeHUUa+gR_kww z>~4n4zDCI0JwMVfzt%s+C}g_hs$6}$6)^R0^Uw^4Kk~;nbcoMaARW8Di2dk-PL1^; z|9vvy|J!!8{vi0@J6}Hy{C~#2%=h45x|S-1#B**wVdZt8sfauVC{S1@n zPbN{7LZU@g*A2Zbw-1ycf#FQjJu_3fm7qH;Uyqw~|83G$DG9o4gZ+K+GcvY-lo!}3 zhgvy=UkGD&0Ohmt^-Dl`RRz?1V}(*PNycN?oj+*J(7P)d0xR*6vN1?wy3cWAlpR9^ zqqOt$;p}%+a42_I#YlnHV)=rtKe06rU62f$14;Fop#DZ7Cf}BQsC|}Zzi{3 z)=L#0ty|U6C=K;*feoKXV#8}4weTan;H*&;TCVw4TvR5pCpX2Ta;HZ+PlZ1{qO(PqQvDarGh*ziTehA-#q zpF)tWwBeV@Z5a4CbuCn7q5Q3|;D<>p=#8TuYz6}g3G|jGj($eDKg-mDFVccr^Yt%G z!M~#gKUWrnLR309*(w%agHf7Dk%|zoaBzW@5pwjh67+ zeEkmy@-r*kAn^Id)m4)&*!-B`O$bueh1_KI7dJ9gX+>17Bq@FEI8G|XtUrHN-`xA3r>Rt zIU<~1pkE6?-lGLSHY{kQNPh)WC%<*;#~D2i)tkd3ZJF8WGCApzvShv|Gq!>vtuVP6 zOqMce<|*FJ5U;U7UulZ>AKGbFL3Cz8bcR^V$V$km&nTdbhM=k-bCsZ}a+Rb;u0GSS zFJHlU5GlF!SvE4URTO-c$>(1oIA8D0F3_(t1vl*)yPl-3x4TViS94IWe1~>?fqsLD z{uAnTj%YNp29-lz1FbDkuYuMSL~b-;6N<)NC;y(}PqEcxYIUYe@$FetfqoO1+D2dg zluXT4W>$$N^*_4d-Qw@v^tmv?Z()_%Jk&^4v-s=C(z;Ap;_I^c1^UenmgXszRHB=u zdC6GnA@hf~kfmEPWr=UbPAJfCHCg)1tnn(v#ZRB>I-V7-##Fs;Fn(g|$-??fS%`v# z#Rd95OcuVTp%y9c*>kJOgY~hoS$2uKV)j| zTT^o?#icm|&g9Cdj_Os`!{q1TO!z77Q0p%SKV=>Cr5)A`jdw?cSkT>gQJ_*xe#{pI z=XC(l^E%ktpw`Ap66M>8PzU{d4^fpuqHGgPq8E_p1(^~p1<}qO^b1X*3r$H?3W=71 z=;rQl)J@%E7m?^inG$7ZqDu#TnMrh!NmQkfXipG*u6v&~gP-j#>;48KEp{=fUz{m* zE->1;gMNugeX&VhrI30tUdX)Az0Y->p6_0b7bA#iSWj~)sb88Yb-oVj(LrBsQa{I} zu2M*yRqZ9p{vu^>?v5CR-`HiOe_5vV`C6#5gMPV5-;!RXcY!ik;X$RO;vq}%a1D95CQ}}If`^_R^lLpl zsFYMZWGfz4kcSnS@Q_nj>pvPi>{+NkSGZ>A%M5kjnEt3z$U_%f)Oo%TM4vCrAk>lQ zSP<=BsJ~zm-EI<9DI{71qBr$GCAhH%D#7bg3106Wdy({C%#{8ypubn4{*p=mTZ6v# zsF~})JRl1`z${oX>Zwq(lHBybfg)L%Ba`rg!vN+C-eUEEBqtfN-8idMFE zkG(?ruVhNU0rdAN)L%8}{|Nfgy$j_Hxk@ApY=dqE3qx+{q3>PTJor5op!51M#TS-P z`5O6oEmMA`fS-Yd`s?85GrH!#$j`otAC+kG^JA{QZ-SK&x^yFBZ;*#KGUZ_!c-XH{ zf78^-Po_;&3K=L)C?wINe2diI%9Q$aP#;vNzwM!}Qb?Vxvtf<|^>;}9olL2p0qTbp z>hF4}s}xdaQ{|!l9;v^VDfM_9ON+t55rz8urUxwP0S`D_d4NhJ6RZ^Ld+3M51GaYW z%b$_456Hs@8S}72@i3uK|Ip;&0`f3k@t_h-Cx91@FNE*O()8FzIXFjgFsV@g z*ulX>#eqsR9RVCnBnSK(8QV$@wr0XXTf}O17`7McB}Hq7Zs>tpcpv1Kkf2gXI$nmI z{V>8aJ^19SYWDg>un4>ai!x|8$O}6T-xlhn9$r+6mzV5hyp)lbvP^kl2jaUzJ>=m< zrFePCO~y+(c`4777j`7RFVs7Gcu^@{UhjuPt{A#8tl{eC~6))NN%%_ ztR}-*iq`|k>j9ba%7JhHB7HP?eVwlHhT$5R>egzDTsi`EEJFlU`EQ_nev!V>qJZ)DN*h^ys}7t21L)UgnU;*1<@-cZfJ8l80n&gnacKe66)dZTt5C! zXyIYJF4M0l;tUlQM{epqs?XVN?+A~KZ6bG@GUcuw++AIyZ#GqOmFXHP#m<*GP4+Aa zKbtAx!$Ejuk^Y=X_*#>&N)f_~l|qtif_k1jJfA5KM}UV_MfwXS4=YR_REqGhMDgGm z2fRoQUd)sOPEcG|q`zcxu+ro}rI3Rjh(mX?Qr=Y=+d`sSG9}6hg6oU)mqAoKa+OI` zrI`TOz8XvLU8K8O-I#nA^SN>h+R4Y`7RwZ=oJw;H}AIXsCVtSX6UQ5&-*qZ)Es`8 zbi#Hzt@Y`OjJohgY!&^=CRt=5y?x+tn zHD(!S7)cGY+rG86khb3)==*ilhXei3wEZuzeYCG(kiK}&iWjPQbBcGP28?MOE^tBA zvy|9p4d0j>N%IcsniM=?A1C^1EK!q7k&+S7o+Z($lBiZ<_%+D!MflZwB6+cf z`^B1GRRU?o;7`|(`YnOK&BZ8q@I1OJNTMJpVL6V_*iEl4L7u#R+!YL0Lu_1Ai@l>W zPAD~B)c1$fdzM7^D2etfiS{)F@8=X}PjHw-oIQ|iiqp>$r>{*MVRtW3+5_z3nws4f zS!rMJyASx?yCfPbi4G`<))@Q_to83#>mQ^IEYbE(W2nK}el9}|Oh~5hjr(R=2bPNA z-V|+sC0Y$qU6vXGnodg%)%UY3wJ$^;2GMa%EqY5Vbs!`^5RxBI68&3AbpMj*h?3}V zL->(SV;uk{4P)tl!@Vie{x*?DSRxI#i6qt<36@8JWn5FU?6g*HlF0$O*I{6~7EF&W ziHePwV=@6Spqitf0vN3PeoTGrL#I=QNCaDEKp@E7|3CR1Sl49v?YVC_3z;0q+ybK!F+I7Oz z!E((P-x*J%tcE$vUeg%njFRJ{-Y)?s?`O=(G{s4I@rdXLqAIP>V5VzwW~A9qFr$A^ zvbfvvWALZg`iN|OWM`|jUCZs*4b7JZ1M>AAL2l_nPi7^%mfeCfTdNYyuFKsO+KKsS zfvr8bMj__2r?5(c+bxg{x(g~sY76pg`FuloFFQ&+KZ2vX;&>j0bb7ZS{2CiB@H47U zbMwM5hl2^Ss#wb5AmBTWw1KM~)Uu3Jb~0(I^7+)JC(2f5(oI&;7 z-VUyLa=solUGoZV0IU|*#LjOc9nO(#E86KkOxruBC__VBt$O;GQz9D4v9dj$M-|2pla zJf6;=(otD)I)gFG-NZQ{q~N%f2QWKT9FH=B^aBF(u$P(`{r+$GJHS8y6aOIRLH!Vp z?FkwO(#1!@#bb1F@TBNaWqe6&R3p>HN5REwf{_8@;-gWq9k>pHvVil?w@*|QC%p&e zOWRcIufM8!=ylq_pf-;(!#CsI(K7JRwk{m2V0?mX0|xc|I<4Qrp^E~(u(mkgo(diD zQ~{naP#j-FttmL3s4)zN9s}STw881f+(0sf0ht>tG7plBl?pQ4Hq9;6#qrm@WDW!H zM-{`;k@={?fB~70DokmvAsLpGRFL5gd+xR`jz8ojb2xxkYx}1ovzpQn1_)PM(rh6a zD-~q8g_zrki{tp1rCVu70CBy`j z8Nz_fI*Uw;qE!^q$`f^>m_qcY##>5eKM`{9KGyk*7=FrQSQh%sV*`$s2`;w z<1vU@lM&Gxv~gxyifAjL18RQv%?g>22VIK!L9xw%dBMmDG|e&+JD!Mu0$bh`iUy#N9@-daO-@$5m~bbiAxKh}2oSol z20rY=dI0nK#qnumshaz)e0}*#;7b8qr=J*<-B|2gl3=%R%}GIbVigmqq2(-s>t&=f z#xxEXUxY>xCnXJvFM>_9lY{6{VP~g|*dUqGHECXsf>)tic8byh$6w?89fzqT9%?mA zD&+Ch%oTveS+f54mAuJ`oD{5YtDS0^r74)Oh?yuBF(1-cQ(>|^>l_!bpfTiGlDMFtpu#7ZWV8GBBw!h+#aXgb(?G;c2o)oWvdVsHD`(p*P%3J zxE|NkGHCIXJl}wV0(hPFMFGv>i}yiS*b{Id z+9zRCM9Uwo*BLnHc%jI-5Ed6N@a2(~#ik`s?_V_J4( zF30ST7if#=yc;wwA-qRp7h;1}9PbZ~abl7uR}LPlVJ1S!22DcA2H?}qGONH033j%~ zWadacFdx#?gWHJ?Dn4rmvF8rLolQ6u6M+)@1y!V)1OMkv#+kwla*{eJKj8MDP7MGo5+zl71H45N=B z32fm$L$;n$Y$29iWU_^@j}vf%(eq8to(7C|aS;1E{;kD*P-9=-(fT}69hc4O@+{#l z4Qdw{p#$xO=apUjIRfprlX}T;3LHeg6f&FdO)kK_F)7GaT)tM!o~JyJG#|7}NC2o| z$L;Ef^5+$4bT@6U?&2>p^}GW8e6c#*_&ep`gW_}P;0ZLo?YTHp&?OK^^clhqbHiVNzJ$@%N!tkPaswX;j1+_NzVI5&Ih=%v9ZK(^errU z)aW|9M)hmNbgIIV-w%N((GR|!!hM7!3DzI;CMR+ef_0+02*_6E@aX?-yvdH-h^QgE zAeA)m4D5nbVPY}kn>|+DEP{NBf)w#HTw5r}EolY$9EeFTdMOe!o-v-U@yV zFEjYHeCKvx)!WGO4j9l_}B!RUry^tNF1A7(xI z1;CNJ&3bZ=QBUp;Ms&K;Z^+f|b=QUs$!f!YNL9brQ5)_P)&I^MAp1ig?vUDWzvS>$ z19R~06ETX#ud_Co^jI4na2VwuuG$a)qJ$IOMG>-)B())%H#w2}Q5*EzQ10&)`t9UJ zUXK8dbT_tAPYVx2+*bW6LVS8NZb}f+bp|ld_0g+3> zd@@HaJMbnu@*v$#Nyu&8DR?!-E;JC@#4aKp9KVpUCzb#aJaJ0CzR~o=3+aiE2cwS# zqmKlm51Vcn1Z3nX(+!_C-0-PjkQr{$QIAF8bpqxp-(nYc! z7yo-!a&|UycEOd_vAA^lpGk=R_ngBz54-%YD-b2D^CWo>BT4+Pf;TylXC2X4K)Nx9 z2JFF`?8s(uGwj7yVtCxF=6T`163~>Z3RmsLAWto@3Se^xqT;6R#qouViikV6bYpYT zldxA7>W_i(MODCjWtGvzy^?*?+iW@4AqSm{;`nvQ8KpIK&|gHMP26|-D6uo^Q6dk> zR2Z>G$dUGPP^-5A-zKcA2P?N3tgPqWk@&6vj=d6$yh<(YM!sGSVsPFmchBabfQY;j z)L#a{_1(2Gsj-D?UK3qZqdd?=g@HLXOD^Y_oqg}+Xbc3Q@rJ}u0b0^URk~?m^tE)m zO$WR#*n1Hz(D%XBW!~M9RL}4D^yN*v$m_xS-1?l@tHH=?Ftk{859VaY-UvouH)&-p zGdWw?nm2{oo`5H{{=8}_g<&gbA#$~XkZNnVWfHz6H1{%S4#3qVVGKzk;oiJy7kNu0 z1b_SRxozz2U_|k~FP_Qwez>NtO+4$tVjE{YpyS&N-#CZ9A>9nl5LgCbm3t>`l^X6M>^M(-;n?y2>bO!(tYzt$3Y(_eG-iYXj!RQCU z=zGEFyJn<25Rlp@LF}#3nu<_I|LEb1pVA4!qh4W;4N_KD7RN7P3To}em8>eGS!P>M zJEWuj397_ut{%9P)pO`FygIp)J=718Zod0Nxn|M(o zU_L})=A>ABuN*l#d>IZczO=_h&O?i}V{D`!6w=Fk6vrO~X)51820soS;DO6~Py)rt zp`^36Q2zkDNZh&Fj5}8|?rh_j`1~pvFVq24aLVXzE2Bd_lB6mf&YSGWrwou&vIi6z zvSEDta(8p8{=4p~!hc7Ps`*UBJp$z@?oqh5kd31O>SALosWVO=%bV=TXA-AzY=hD; zjDlku;LoqapSN57Ox@;mt@&ISISwVs$aq|}FM`_F!9?NDMXjGz;hh^JE_Ao4iq@Y% z*zLh7xw#$1PIAOM7j!pZz-K!wUjG8h-BbllZmJpumAuNWS@QPcgG#u{7eM@h;Z{FO z1fED{v~PpjMTPoyl>LGY!bQTx1pvOt;NpU+k+JWCkspEpWq{5uY#xfV$oE10Ta>z> zN?Vq?e7NRE5quJ8L6!!}@~#Mx#bN#Y&1Rw3T~q$2tc z!N`yF#aTuP6-Bn1LH>9GXGcDVOEC6XQNYoY;1L^^ui%IcNX+5$x0F~7pE2l$YAhI+ zR-mV3gKkS$M$3OYqNQDQ`qEs3d$ssq%W;X@Dj$rUxLw} z&FFVBVfYCz1wwzeUYEGWZNH15PGJEsek!hs>i8d8;xr^lbv%r^{ zxcUctBw1W_^0@+05$-ms##c(F7O*s~3vs2@ExDam$JfG~Xk_`<-o7Q#ulMcS z`2gnJXs#fUFMOEqWpYasFBe3w7XbDczct^9w8X?>Yu7#X0;fDQV!u-Tz*201VZ8fK z6>7%7QdwK55?L2~So-vPl|HZ(GgCd$0DQ{^K-2n`8c8U)pr_Gc_ZP!D$gYZwC@e^ACHdN9k4=rJ(TU?Db55>NX*BUK8gSL35d zD2w&lXz0F1k7^EDR4u#A)|O%i{KjqJ7JVhjjunk)vL&@+iA*goZQ=Ku>_}bTZ^G%C zu`DVxtLzwjZ^_UF!uYBRn{rzlHmO|sa%{fd7RtSe%3WtD7qubLm&z%Fi9sFLL=*RC zsD)w5CHZ=LliJOo7R@bF?M63j60P<3U_*!&4XqYl;v@{rg)d1ye`Vl;fo~E6pJPtu zoCEuc1nkIY9qoqJ^2^YtxS^+>TbAhc*A$igh4eZ(=oS3e>Fwm8*C7eL0fl9eqB3Z< zC;lRd0FVdU?#k|AHM)j{W#}3fm$hc*p%=T7!Ll@6Ng4UGD~a(HmgTZ5X@gJEmE2t` zEpxtM)5`2Ja_lSb*!tG#r7&Z2MNNp(^~2wuTDLN-FIl>&CwsRKp+Cg?;`nap5NZ7! zcr?ucdxzO~>c_qldbT(9WY4z0wuhUYJsoIFcJ|Wxmtl$E)^JUkzK8I|NjP-}R@5Zi zQu9@mxVJH>vJ~!pW6y<9l5M~aG2p6?^c=<2$e^(`Y#>K}YHOH_6J-n>(+0TNizQ`m zZ*A`~eSq+#-fDK$*2>E=m1c2he_6KPd|B3uA#)!G-pfP znxY=2IEJR^Ql=f7h;fWtD#q4b~`v@ zm$z=1Vgh_=Sl_!04Y8aIEi=LW#R0@DJH}+IWlNp>-6J*F)rJM`cdqVa#bf z>WV7=r<$`foxHiRSr}@8bsny=lrxU*cP!pqO(=^%FLD~O(iSR7s%h;&xMm`CcpU!5 zV**TAFs7$l)S+|ElVNt$BAI4()PiZ*?5H-$-c~ufg@}L?I3YB-Ou9^L?#8uIt4$}D zB_@Q}i<-zO;mMc|o>;~ge31#PrI-w}b#|PSVaKsLsaF~eNh#%k2HI#k{$^6xsWxGaRmoFb!p>rsaWB zI%pIFrjDiT)O0{lZ07JV*-PY6HOG_qnfQ|p7MUA)Zb~^}y;|+tIi8-vHf+4?XM!Xzs~t;vDFt< z)LKD0tq~wUEr3*8g(cJ`m^i?zw7q=}JzBTxmz%+S#QOfSj{S z59aGvmvLt%JRF-e)$GzWHr;W9WwP(|s#2JIz{h@3_7UjLw(}D6NqPmo37xP(OYk5P z-J9e~$(T>lD?IZ_>Ayv6XV3L5VjPX8U93h5#@EH~t!!{%!#O{;p{SuEOW5nJfkStA zigvBN&Z>g}P+5bqYiOlw)U=i22h&zt?9*1Qwb}~9YS-lJt6(+9$F`Hq33|TOY?Oi^ z_{=t}=9&|{-YH0mIl=T;VZ&C;33_nRVbItNp66iI^+j;b30_C#UZ<3+bOGhwYgcY- zXX&V1&Izu_*H=TikFrWUmTXRN4FEfXI<9}dc9R(;oyD)xmU+wc>Bz)AS}f47_ad+ewz z?)R+I?{;vnll%Vo+beVK@3oV3b6?}+egL`O8{F&U{=Qbt!VY9F_JJ1lVh35hSQ~tb zUhH7`-j7lb0)cBS;)7Jw5d00(9xmhBQVTG}+S1{5+GAy0`S~yuF3Ut+D?jlz=J%eQ z=ls1VYSxI@<7JU2P^b1sF~me!_lcay<7N6|!0>xduKP@nD_paYT>cGnSasSOTbOyu z?B_hi*6RV)!xpsWS|NKqx6eyGNrVUDZzOwJPuY5*##reoTZf1tVH5|3qcJdiQU->x zC(0rlF*c;RYq9F{>9PnkDXTs$1-dl-3`ri0ze8If1bdbiqGTO*hacJ{+e9*T_^YR6 zo4t~4c1bpdk{t%gHi=}l{=>1_^Vzb9@OuOh9hrF*dM;fRI*N=w?`8CP7o$hZ8j)wq zWU=Sds9&}IahS}&yacBuUhs7KpH0%~Kcl05AG}Vcn_iGk|8wZ{zrapEG_E2bDoHYp z6F@ajsQ|?a- zDRq)<^$Ywwvu*%C8ie}!TA9)5K>ZvMeWfhAr7ZfA(HWSoz0s#(~ zy9Md)1+k5F`kSH(wHP*B^A_vGZ2TRs-E3oTqZvLna+vfAiY(%?N-Qf>#g!&sLb};i zkkuluBOJ_S#pmq9Ta5E>x_vMC%34c?R4Z%k!hpDKW$lSv&Z;Mc*llzHJ!w%*aOwezJ1wV^xm_d{h?M3JzrO zF0+j9h(*ABs#(T1U28s}on)BLD@XPe`#mv?N9=bJV#hi9Pl)~@33DcZMv|d44D|^N z)iOr1yVh(IM$BUz&TBa%Ja)mw$OR@N%XpI=+16r4u$=f)VdP?zq(*qkf%aJ$mlIp+ z{9K}#cR4YSH_*N;+wpSZ&!FDcEGNEcV&_9X+)lufP(Bdn9GsBE7QLP*$b-lui=rGMbO>2pvo7n z`9U-lu-5aL!lzmrkHsddyK}KgwH6F%2N3O5 z5u$xu&|VcOj$h5!v}~qbXiFXN*8^VAUT-XIxGF-ly|e>ET!w@36~1(bV==Xnq`|X& z+JqRkY$s@#IdX!A`2bc@obA(>r)P`|0U-6sJ^kt;qeJ+Hyn=I{8`DrMZ!}&8-kp03nwIQI|3x72@)sTTgi@#&iR;=UPJ~NI!Gd`h)acxz_8t?KMEVaP=1n3`2pV@JM9ls;xPt+zPt9)jn zR6g?|b>*A1Lj@!1PNZ(sL5%65?j*NvrcgJNZMw-Z)>^o9GnKlT2Hi|-nQo@HJYF`C z^%?k^nbxeHcsYx#Pw}!o#l?CfS)UEo8_4<&;^iD|YBH0ViDEMIA+^bxv}qDA#kk4h zU)4w@pN_vX(kWRx$FsUKHAOf?(jA1e-L>vaT4j#SD(#e2=D4gP%R6WpS%U%V zG{QRaia^N%g3gI+d$>HF0;)M3T)7_Ozb@!px1h4{gTgj>g>7;R%Q++wb_s9VMvkYj z`hjM}z^dDMB9fd13XwSFgDy6X!Qi_POCzp#A<&WYYz(81A_smt(a#t3oI#@II2ZI^ z*VqXjja>+`)Yyq$jh*P$*hPSqNer2;Y8RPrG@@}%LgCH8Nuh{R*(G?UOq^&kWNxk6 z!Zjz0dN`#-RGcVErygZEiAsM(9XiFMC{8|6QKxzpb*fuYa*`){8gb4!p|^3eC<;Dy z5_}BFoP*MKcMR(8G<$G&vXeNAL1k@3PD9|-(TUBiD z^4;6FD7=3uLnQ45g+4r-veEVi*5v!-Nb+?*k5oR4Q0|gJL z)K+%T&Sp#UvGlyZke>I)=y`v}Zn3t?lPkmMFPMqZ`Nz^{mIE%ahTYD~IibkkLq?Lp z%Q+!^33;Jk=aJ+;WQxF9_q)}cD=e;8EdGNm-q1lU$^MddK9aFS+l8Khz_A|u;zWP& zyB?VezsxIO2J(go0KBp04(rf_JJ6+9>X9PwpRHl-a59I2cUoX~5==NxfQg&k2If$Q z{~n5*ON-n=UU8P^Qp%R{EKgUisMB?p=Rb+)JYwNYL{`W(6Jh2^SI2w+$0=qa;yfXf zE_+`3*$6p7Q^Q%FHk6*TJhO!v5KG!+;MRor-6V&zJZ*wH&++z*e*lHB#`ydY4z%Slp3)Dot*iFT7aG%9m?r!R-)*rU zBkeftGaw>^Wz59bTki{n`|}k>qheACqP6yvH<5V#2SrGsPE!JU_X*Jzu{R6MjE)G85k% zNVUWC-=3LbRSJ6jIoV9{WsY_t)eh72SWz{;mJhss$BjTaS;g_Wc5%*@Px9u&qXUCHthAsT*Zeuxcdf9K)Gr4e^BoSR z1UkzUpxWyXak{X9?uwT#*P>O^R=T-gWk+SLF$&k|D;?ZlN6sto*DZ2Am)jjB`)~%u z10@J2YaBQ*b%Yl62BSf|J``CEF1kCJ>OrQez|?hMN;>3<93)NMAzw=#WH)H!Mh_1+ zC*k2HgNGYKk#*oTVC|8(?@j zp{{kP!!`e;efV8e?amPQI^6<9N%lGk0rxV7{&o7D0AE&7a~H9w`ETq+u_?m25a3QW zQ)(~D0QTf!XzK3{MWpoJz_3r|wG0OrVHu37#c)j5IvJ!SJGck~UPFTbNIIIm$L^`_ z%DXQcR^8T08B1AoTcdIn>^4v;+1=9n-VeRjJ3RZkFUtKRr2h-Gb5Snh$D&*#34QoQxx=6reqSy2ySz8egpu#T$j)G- zq&y*ITP5Z6a!5J7JOdRHIluFISh+rduS%h3 zuvB^muH680%I)t3Q}y4{V{Kjk&4Vk^f3vfybu0j9cgeZrckw94lGto2b9TA(-xMhN zZ%eynU}h?pyKmYR=yRdmOQ_sS4dog=Bef!>shmBdKm^x6UoT0m0@U&f^b<^K%h_9; zCr)J@528*7LEnYnkwiZPAC1IIb96rVj@x@?h)1CYc9{GUF?otJL*jPRYbTYXYU0e0 zdF6@TVa=K4e<3~R86_wG*XcRWC^_Z7PH%xjpQrxU=`D27JMF(tZ;^xE=}G7fIHNpr zW;tB8H~#u0y3Z$~_Uz8?b3b&S&nQRtd2xAby3hT!v&z{~Sq$Gy(nDlN zNcO+)r8SqM)3Uq=*WfSjfv8ebr=NwQ=8pJq&DrFC0RCd?cvY2~`gm3QB*GGwP1T#+ zVKsG*+0Lh02WS_%QV!HEa;3Y|LgRYJLp}M zgx-K<<&lf&zjDln)4vX;uN;E9y9{=$(=U*^R&yDl55-?yoqnN8x}sdaNbUw)ULLuE z`D$5%ckN~?Yr~ax{@u0VFekT%vo;)o+Hg5*!&N(q97-Bg6xV=F_2A9Oab%r-mBfjf zt0~q|_#3MNgi3B%fVhTbkHO!#Bm<>Zsszhf6=|@ZW;v?`)57Jf*QV3-u})1N$HQC3 zo4Xd=X%4E3{nXD+izd9Zd%0#15E!CbaI0S$6O#R@n8K1wO4 z+gcQyn^kjvo~|q=F(%Vr#>ywp0RVb4e0_NjrBiU_^_}(RnN4zN)(v(BN7Y%Oe2#Z?Uu1$<7+GGY^0BMIVFb6&dWDz?Zov;r-;Aa`Bhd z;x8xTkcnFVskQ#ouwVgAmNw7UAU-2o3N~V6)H6HbMbdQ~@o^eXO@%;V%PQznDhj4> zsU6k`hx!GlQPnN{>0Q=rUe93MR*M?4BbTw?g=xr}%F|3kS_^NnT--Pd#0H;YZc#0E zSXb_v=uUo~oh(tP)7Pm6<7S#hcC+v*Csg2UyIaU&3(vN*!KOIdF3$B0+O2N(lAmuU zUq02=mtV!#hqsmMw+f#$sk*aFYMtR`w`TG~hi+kW>pA1@wsPn3Zn8i^+u&w1`8jv; z-Bj)Na(#pFr}|LlS$8VU;?z2#4JpoQoBXW1IG03dceoize%_sY^;G+(($`((`W=?O z)SbnJrLVMS-reO^$|d9lr?TBmmq>Z;ohw(~Avw>zQ~QHjclO=gw443xJLR;NNS0y& zrp~@o3qD$O{+)>&*fyH6H1+v+_eec)tOaSwIe0cAJ35Fh@flWGvyl_IhwTS6ucy_# zD>s?ux!6O_#QP_FV+UvA-79k1_Yt-7Y&_j=GhID{yG|n{ma3D{*kSRbA|BUmo!EhLfJFXZ2NE zd#>JpNwck08O&ijS1&FrYtZqN1r6x@Jyafn-ekk6(k3cds-3SmU+-bj@$H6=WnE6j z<4FvjEYfKPPZo@~1Dv#@?mTDDK9!2juY4GnZ-X9@qo>ub8)HAkBjt%*H=He#Wj?ah zkPC3oLw#8FP#x<1<2BfMcG!7r&2q@BvBo0v~m^?T(Z4thf5{ z`>AvNQEAd^9uqTgT@GDcR?29V9gJL9(D@?1=5Yb%LK=c|ZH)Frx@yTTmZAh)X@}zR zM}@5?92Ie+kmE8NlH&pzkhA-3Mnli^j6vhJp=bOF5SK{3kvHvQPnJi(oti9AOt);Z z;3*O5NkgQkDAHfy_lgB5IgQ_2FfH`^r|knS_w-Dh9}A}6KV$QI<305=ZohwqIxBz) zH}NJv@>F?!9wr!`L~EDU@1L;wJ-vTVdMq{~}3i>{voJl*Gc zT(#%QxuC%+;984;A(r}_;0r9Hy;#m;U!H?YCOh^8!0Lm4F9JCBQh8(xtN2T#@*s1N?7!WOQ1S-9PT@<8)4(Y%tbY6(vF3X1qEC|9Sy;;2Qh3gTCp1I(`} z%zsHOa`51)Ft3A7eIW?G!J7`T*UBScL{%8YU&|`Y>%#q;2KR4~`@f>XC>GLJ7z^ev zsW5MYbyk=+ycOmRcZGR}I_n7Ky~~@T$m`|xh4mdGua(yqq^&To*ec9>ESewNQXY9( zsvEFH8pg&;72RRu#Sb~x>F+tJ-TOlBD}&tEB$u({dO5O2;C>f*Hv*L)-j_zop$#_w_c zzf!=*X%+ATIH3ZzdKIwMt$-gXH(Q;5@g^to(T=)m@7ZeLPb@=k{Fyh|k$35h8crRS zj$YDJhd&XW{Q`LE>{nbf*V(qT+V~ALsEtp(+W6G1jsH*%3gq*PK%a;}>EGd=B$Y2! z4XXJ}1oi_T1rFevxxmhoc-Ft;SIiw34!s%?`PwZqb|AX z(oS94s|&wp4G@0ATKmG?XU%#n$w^(aY!lK?$%ORdn2>&o6Vm5HhZ}iP(?&U>_w;ls zWz;%tM*yep2;h`-0QQ+|_%isyIo%+e-qZklJaGn?eAJs}__937fxkO!T%6~CeF1LG z!))y4wx0v`rQ5drK)7vj9tLQMML?mIzS?m7cVBJ4f`(3Aa(i}bn>Wm#mzPyoR0db= zFtbz5&O?TGXzTa;3vzvhwSlkD_2ZK9t_AuxSOm0^OODp2UeC3{vz|+p!s4El$<}jy zYp=m6Pi{$%wRP*cJh<5WtM&MHR<&x^damNwcCP2zuGVuYP^{-#naO%CDwj*WD+=`Q zq1+p(+?%ZRTvChGdafS;*csGuO_;LmPneRYyLK_)8-{s+9@;PN@%OI|t6{hxduS``^k1ZM)%?cLr{S+E@*nh!WijvXDku&3PkH1I zNYhihILYUfFT>}QWs>i*PKYk3cnV>v7z^$q5g)#G#xDW#ybWKxo{L6mviJqGdix@N zFRo<$i|o$!<%J{b+oMmJ)tN8bu;$;`dN9D(ITGk>e5HvC?0-Y()Y^1*y-sTDKxOKv zSDoIbv-H_=IzvBPc!Z_y_;9>Rg`lpHUM|7<;;)}}WfFy5X)5eWDhvZa3y*66h#0j= z&V<;5jsBh(X141b5#;J}a}w|=PR^;3^DGizj`J*N zUW;2bgK{g5>REsdZ#==`R^fI)3#88LYzY?|htILeyU})?`7V8LHgN-CqnysLQNG(o z1xanRU!9(>Y}A3uAB4XlVk4F6wNW9#hT?CSrrGM&dY%Ep?LoaD5^ZCZPi33ow9N?E zrl@m7knRtpf8(h}of1m)o@$iDHa7SawmE>dv4MGPV{x0xHvf;f?|_f0*#1vKfFM#0 zf@YH~fg1+0h+x?)pkNRSC~ATQDORvIP@nqLRUo2bqp8>%7A)Ak_TJyK_uji=*Jt~G zzh~y&y=8ZkxcK|?`GkA-&diz9=FH5QGc#Kkptg5bCgWY1^c0y)piH)fOu9oR3$P>u3PqYUzrNqS?H9&nING;4}PfFKdLC%Ok7Qs4DZxaCu~v zJ)%`EZ$d8bJ+X9#wP|Zy7GQa}yqp2=!}vd_m}$M!mDH7F#zO@5rd9l24<2DmfUV1E zNPHDe-p79yqG1Zi)mLN4-HvO+x!|)8cxLKwWHP3ADGx+kRe|Jvn>(gQRjExN?gLEK z!%ysW2z>9o4nguk)H-~#jkTHZvxuxR}K94>0cAt-F9>AoCi^sOqQ5pD+T z+5c``mEGykLGN^!LEtlr0X_!6=^Q=K1H7vSKA*tndx5uiusa&Sdjfd6n|%t*#~mDx z-Dk#QIgTm3Jv7+Wr7+mNBiCSefv>yCw|Sv3)bk^VBNY zyr8pUuzN=_gIzLrZ7_FaH~Vxj_kJ?>0mEDzV3_R*&Ok;Ul8EQu?+I8Nx7%#e&0gr# z_8@zLGfhtromd*f{swIM#6|{~6<&0TL2C|wV2FtZBE)=up$!YmyyCVNvqywK@JtvK zXFF@^+^W@B-+8|Jmi&YDX@BSWzWSE_&tJ)LLpOlcy3n95Y;rxq{ls}wb2iwI`7ZJ4~_**ZBC#Y2|^T*rk-JAL9 zMbirPKqW0P>tZhpeX5Aw#j^u~40CrtTttRlQpFt&1z518APX=OkR1&hG})IRV>Tk; zQu1U&{B5+lc#;p1d_1|V5KlJtD^ZF{)Bq*A6g<%@KAN1it@vC{V$=8=@&Ak1D+&=i z(ogIt5<42iUQPvAUUeSBp{}a3mx`o0ebaAXtezscGwOmdb5~XQ0}01iS8LEKs_e5} z(E6>NpjQMykHsadm{4&m$F&%L`U1N4Jy&e9RuCBfg>lYnm-Z*;@-*t^tVs5cYh)O(CMkr&`Fm!X zz0K<t*bn277kwd2&_0UOY#n9>LVuns+E(g8M-RxVy+!x8* zm)xO~oQR<((YGQa_kH4-6=i$qrUp?{m1Kj%vaxidFtCOWN=!gZ-H|I#{x{Fr6Z9;vFYEDTTdVNReV1y zURh!P6O7gMJ>jeGfjoqUSN~fjG|%e)lfL>M%u!#%LsjZwces=7Ywt!g+L(Q98h!0U z>}#J2Ix3&`0(S%6Ml{(^K`{{k@E?L5i9Z}fJ+Qm^3M1inCcEw>oR#=rme9vghu7{~ z<$-@J`ivJlpGYyP9PKA`3|*X?VZHgka0Xh>R#_1bx^sB!*_hdtnK^U<55p?}G&b4K zR^gTQw=~i4d{y>m=93O3CH+&+RjKDm&RG0yo?~RyPI6u-M9w%rIdUEQmLTUjkV6^b zj6L-q$`GOGFVa(&cN0J$RUB;oO#tn^X#GVhop%GkJ)#}%AS%28ppCb$JAeVH4Z;5} zQ5J2#ecc1RwWxC0*tr8Z{1Q2g=<%1yVZHMvNA!3gRXFUA9>-SDdvyzXg-Gk&pM`c^ zgrl!tCeyK72LE3Y%nYu?vdeJa)gYFYqmWO9JW4`{O}cK151Q;R#T2Jj z5zO)U(|dn+=x2RV1=V>TeML*-v*;^!U|)gmNauGAc3{LblpRi4Ny)>!4mC5^i5WbJ zFXd_#xsJ8^@5uFhaTM;3Wm#UFU?GmelRwW|yv^m+_1r`X#08jjj z`!rppSt^$h{I6B!Y{~*q@|;a6dv2}Ys&H2>V?zE00NGZX{-a9$Sp`IZUhfL5A7jUc z`lHJJ6=J`>*8WW%vD9C}8Xm?Dj-E@@3kXOGUssD}sDp3h&OQX%|5>H}s^Y#Dz&IZ; zn!BV+B1&`p0{o_`@D9D=jcjSevo7tk1XM6?hou=Pi^wEQETY+vRu=JFL^)tx!naBy z5)bsV#Gc4rS6Yb7<)our{YWceJr^2|hX(>*n8_9N<9cTUv>YZ;?!_6QXA!dF3d5G= zc;e@kcvu7aSsf$paBSXSXPi2SwkLcDenAuhM{gqJM0!}Wi-8l0g2k8Q3_C#}0%AnQ z^e*e*kwTG}c_49o>g7Pb^ABWC>}5WJ=I0y~c8ow(G4+B-E(1ZFfK_H{;eN6@`D8M* zrRBsJ6^O^y7is{)!;Q5wpI0n}heuA8s90aIl(_~lCBlhX#XYppst%Fn4ytp+9IaoW z)wQ8>M0JTE(+$J|?twL^40+uHfz%wmvuspt=YI1w$TF{s-~s6-z(H-0105cjb`#LW z<7WN1ML%xUkK6Qvn?8|+8#t}55$mL0wmbfOQC3l&$12K;SVckKfcl@R^)i~n!TidK z>=?moH&+2)5doHRmX?j&c?XoWd6@6v%YvBIJ(3;korHGE!DwaXB!)BrpZn_s{WSGt zdE-SGf_n!*m6MFGO2-sypSJcoqbwG8FqC>;u1B@OF4e=)lAcCOdPG#Oh}|{9Ty6}m zY=@=kTT623lUdE7^lJr8-`#*awUc`sI|hh<-#gz23JVOaz7dJ#Ne|`f!q7+&5Gw(( zHy|F_%^nuvB?HhsTpl3c8EURJj9907()>5+HEe|9m9VgVa707iT&ay|gxAJjU)()_ zdU)hN>dAi_Vn2?caT0zy6;V={5zJL-))tZA-;^9QUX3}(h7q+9+Ba^&Ob;hFh8N3? zT5#iE-RyLj8v~luDB*^i7U0Gvz)#|=txY2a0ClqjhrIrtz%pw~Z0M!E=EYpc?;25d znrPJn$(MBNd{IsIrV$iLjV4D2;csx7mDZDANz6D#gD+spsEFD`m@*_if=n4vEK`Pq zDMxp+8y%*6&LyES5go<3q1xPOHsegQ**v1Q0A3q6sjY-#ZovS@wnnBD$Y?O2v+TlK z@6rv)Gw^4GH9iu`k!9#GauwM&X|l&hd>oz@O-&%HHpSoQ5CmTxW71mz8Ti8CwFW0A zfRjz364^RMi1wHWLILG9n%agHjl9K$WH#E*~2BdAYOty-stwknVBN;M5p1*Gh z*lZ_4CdYKMCpt38HrwskoW0+5yy#wZn*WCOymW0BKAZ5luCrA~~gxZ6ZI& z=E5~jf==!bQB6Ysj_K`4|8~XFzbokfcQ<=dnEokEYG+Y9H!YxcyMQV=NRJoaWg!G| zO{PGn<8Qb0jwrA@1-cgmIvE0GgPqiRgWVg1?;KIP2;nmT2807vaf06;gdf|@o)RYf zfF?Cv2zS#0B<%(|4g^I9HCd;4Aoz2ob|>Knwax=SL^CI~bH^ zRY!pQ=@GS?kbh))D#@Q(EcyQi`Nwy&_j1Vpg0^T6V~dV$Vt3`H1qho#!j8k=@!55w zy@6*YgHBo|A}8{tT+IN+)=U_)Y_zZVD%3uNAif4Z=OH&)v{&ma%CsPFlff)+tF!m; z7?k~`F1j;V*L*2I(@iIgX|dD+lIdiV=@dLknvOpZ(s=Qj%oP{#rHeX1W?%qF?*oLj zKec&gq!4CaF3=evS5)mQik6Sl`$W`!WGuZ90bZX)2xSV$71s>(aam2)iQVi&99{T| zx^Pe=d7x2;3}@%+FsFN(WxA)sB5F3VyOF7f3kNx_CYw31Ht@1i!dtd8lFN~rLU{uA z(>~CZR5bN3se6UJ2yJ~o@qsYW_CYK;34a;}-?DoVuvmPQK;w9o(0J)lmcxgV zEIF-(?g5THA@ZXj^7Ii*xrZOnN;CKK1vjce_^pycKGa1Uj)}BJGczWlsiPz67(wv? zR6-OV#FN7ZS4olY78@r&hdCOtxq^B02XFekl_ww>x76& za|HKS0%4bbCS-`~jL0(KUCO|`8_L@!_(y8H*9N~F9ghS*jR|F%9FJ55a}5pUYRHjm zdbCR7?K0$Lcg;Aw7s{@&dI^JN-}O<#H1BuTpwfwVr%P1U;XHZhdnRExWE@O zF7OGA3w#mA1-^lCfiJ?iz&D3+bIG`MfDtqf??p1M27sV(m_?qHgK<%$S|>+LS}w-b z(^Qy^Ihh!;1G%_W(ZGjp(W=0gE}<_mqd4P+HMk5ucd2yPGg#mbbx>4ptezgRS6 zNu=Pho_-8vzC{u16yuVELvU3ER}K>8WpIVawx`Vc%rtl{2LF^~j)tSiUW5UT*>`21 zg7=mfyeBO&t~g}=!auG$$PZR6+npA&0)R$7Y+)lJ!*tbH!NO;^t_}HhS%ii0QO^0z zY{YNp`M+(z~Q5+Y0 zLDmWE4^}XOpq{LW8N4xXK@X~@cs>v6NY;cE`wO!*26iOI#}A+N$*04@Ao?~NaJQm0?BnZl&VRhZ5X?alH` zf2XiC!yhtZ0j^11%C1(I{UpJF6@O>ja2Xrd=FE!cI6kc-m>)SmqpI~fq%yNFR4qF| z>lRC09%(Jp8q7qHTNuz=;)$7rhMeCqme~sV(w7pct??!Ul<|DQ9e5Z$qy^mx{7{Zu z>N1&lWhUZ-bw$LQ=c6=mZKr*{v}-Ijcv|9iDA{CRLCNaojo-DY<%0V5Oamf2;5n^B zukq>Ah*lGR$xLcTzLcxwoJslibPBYvjWG2?^Yq@#%uWEHu7tt!-~Z=RKkFL5n#l0! zDq|b3jHs)L(yjnxt%$ga>)M_vOt>~;uMq2|RT*u>S$HWm3opnoUZ~7e9@AOeR5lcv5JT$(K%QrHmNU*GJS1By?|l_3@fAlM@0AFfbQQ>qbA%)J@6#MBT!gXXdII`*g8oS2I`Fd~2ZQ+jPzQw4>(RP0hDO)E!#8_eE)5 zm!<+% zxS=ygG1q-svp00K{%C@*9}>Uw!=k3g7@+^g(;-V^fodI_j~o2P;6u7I5$#+oqWyvB>k+%c zLG)@O`kG)#0^IK)K;yOoxMl6)Yrtw{1!Rd68Z>Uhfm>-*riNXJVV7bt8~_a8jM!^9 z7+y&X-*7Q>zlSiCC>Q$;V0cLg!!nIwS7O+;SPU&-_;$oz)4}j^V)&Meq5D0AVYyiN zw}9b~Aq>kkhTVu^w_-7@1BUNL?CuVRmlDHwTnydsAq;gN^bRn*EQDc&#;^x5>`^R+ z^}z7ch~3k{@GfHbNhJBPAUU(*StrfcV9qrS=3F+IPy7w$V<3IOASmSpgLH%GMMQfQ zi)a#vejc%VJBZpu^fS#$5a50X0lLB5P#Jn6H50bvV_;kD+NWeTiY;Vlf;F3|B?$eh!8!iQzXchVJ(ehPuIg0}S5_VQ3o6 zI>d0D;xGhye?{yn2fcfV-n!LXPiC*DzhkDiU@+%o=!W$aZV6Oz!gzb#YV<5J66l($ zuJ+v^>lF1E^zo{?+V^=36Vnk=5h;d>$IwsmP>c1d?JYpaqZ~2307B%BSN(-Za#;B8 zU^M%81`p?d=5p69S^t*a`rA89cog;Xyt;?G*;}Lj8yQn^ho$Scj_GPXIwOTU2OxPd z0<)Gk&PQNYZ#ZfuNGO&yFPs&upC2RZSwLJ7mCQedp(Nf1NXLoJ7RQ@=%$v9~{oAC-g@I#FZ+bjs zGQN5$W)p@+4CQXLJY%W9BO~6X{75%@o1lzDGUUEa$qFhj_s7icHC~qj@3N&doAZ$e zIpi!=m?=_fVreYBEqStSHD_2JKD2jCNnK?p{zNys$>GD@(ADJj)yeIulM|)s^ZdX` zXi+kY-?3V>s$o*K+Nqj%*IwM++PNA;eyFIJUnpwShpIC383HyZSeJxSMSZ?tBAh-S zDms1EvkvNJ?Lr`MApyuAu3GNSg{bYTIp;yI))KR~qm7Z_&BW~23D&{B^8RHbRHJ=M zLz}FpJapb=p$+vOTA3Nan7du7QNhnv)?tJdy~)*>qpgp>;fyNVRZ}hjwPn5J{_$O_ zvjMfu;Kl}Yv1|nQ%H-VNENYN$;ykTI-RB%utzWn@xKsK}Xg8la72sqUk%v0hlCSmL8Ue#`R&xpOj z2iaq)_Qjh7-Y`fde`Q;y_dv$2IQ`sO?T1$&Fac;{+vPWYQnKPQ;KCx?p9BVOyW9;h zx8qv-_E>b?zZ#3KtZ)Zb*rX1?D|a+vzu{NyuI)h9Ci@AqD}JnP7uGfzwH@fxwkvC! zq7FjhRFs(}fgUbT{NByf!2~N2E=|^t?MCVlmfjtId-##slUFe8h3gj%sc!BnEZCc9 z&H$PR15F}06R~|Ws;#}NN5ak=R85;Uc+>&%95QM@d5##huRJ%!?+I!~H8pTySv>;aby4)l|WYZ7r!A$%Bz z0=M*@U^w?g?NP1xI*WOA6vl37qRnO4KBOSA;#%ENXmwDY!+ou8mT7fIt0NuT`7dKT zkE~Wl0hwc)OcC-~_t-QiUXC11j!0w-IWo&>uzrpl6Xb}*&1AW91nSD1z{PAvU(ex7 zxjGu;Ti<(-{@gK#`*CGS(?Z19AAK+w-2Of0`O3b{`Kr;O4<9CPLJ zJ}?-z;5Z+H|7{p7TX|H*Ve$!v$(d?(A_-@FjVVcTbf;|p8?3}S1qU#)P60jA*Y6>0 zh-s}v!b$**sI{9p(1zzwa@<@-lo9EHCFvxjG40dK5;F#{}n~ z!d%JP=T}>E9o5V#7)CK)!)#%&Qkgl3gBY#5fd+*zckO@WZjL54T8VQ~S_+GU-9!IUe_0!+9T6$lf!?*dFXnWDlNp@67X z5;unRxq1BhmpVmAxRKdF0wcK!i0agkLfwo40W7(%La^kP3SfB(v4jzG4TIl^Wx3@w zV%e`=Bc_S)7%^#cj17Yk!+ViNj1gWiV!-uG$A|&fammxGlc!ZD7dWPjS8G`d%}Cf@ zppDd(o&Kkdl$#=psZO@=w9FasN;sLdC_8SYm3x4gaBxTHV&-zRaq7{iXlk)Yka1#^ z;KO(d_aEU)d3s5;LWVZM8XQe66xli|7qe<-6IUzM;Bp8S4=VsY7&0J)4u4kxz7ELgt#bl$y*%(_u|CT`i@cB97 zg^t6;tH4U}^URBe<>n3Bz!#838kU+b)cz77=I6?JA&Or{0aAQ^nBwz; z6u*MJBJH>Te=nlYK$8cjYw^ zZMdX5@U1SIy0}_hBBZ~?GAPUIkx+-u=~T>%k1Sn@t6J~kn~?DyU$6*YtuBV@2C6tF znz~HjzmHsm{{fzchGQM@AM#rO{6~a}VQ1!JzH~}oUabHq2!3d7>WUy$4RL&9RiB_F zpPv!@^1K$nNf8U+{3u`nJ||dpRWfe*sI*b7zharzS|F{; z&_u(ELh9pU)6hS4O|@Df%=y}2^bMYA!EqJdn!5wZZ~4+qT~pn>CT2BPRjaG3C9L{8 z7D3Ofd8va){5>++;T%`Sm8UiOr>>RSzGqQ@{sB)kEH4DV!MQHfw0=YpHm&QkIzG+J z`>zi+t)Gy$c}>*wGhe!@Yk@eozurLM>*IO?+h6B-WEcn&k=YOGeDhnZhU?GI>aC)*)w56GP>d&Pl6RgL5HGJnRwS0#P=s5j!utdu-j=Raa z(zgwq``SMld?FmC`ZzdD*r1j#vc3H|nA^aw=i&h!pRzOGe_wp+B_WpASJ4FW{;DQ> zQ8lusUX~OFi8F2djgU7ku_3)0ZiKG})x0P2Hupt6z4)@Wdbzr} z56iw*ZFTfC!mpi1cn}&PCky&m!`0gcUkf7Fn>bYhIfj)tua$niT7hjv%Hd5Pzfx}q zw;7pEZZquMk;7Y(!U%Mx^u@ECa$vYRdHHr&4sQoZl|b@!P|vz3)Z97(ALZmw)zOTAv*+*=MH z;H{d~OzNrbs^+bl_#NZ#Zt6~qAtX;%d}DMv?x?-Jy17epg}SX;hW(f8VgIEV1UK0i zu~js@E7D*PJEg(6^8#b>R%Tr?LwV1q2OJ>}08sM$eUH4%4CVbG`4-Wp%Vi*6%GJ9Z zuYVu~-NEo+BKcun6Yivl33q-JFyRv8ju_bwtC6X|%!d*0!67k5+#d-8hcFRnGag(QJS49DgKDePV=lgQ%*6pP7cgBg ziSTNA27(|$zOANSoYET(bmXX>4fxVoeOzs>U1f+prXrtIE6oCi*AxGZ@l>C}o`}kQ zriI?{X|?*CgL4V3$EGcpKmJWyE`Ma$(?F1|jmm8)BjfI9>I-3I+MvQ;iHMR{ekpWUkV{(_yrVfhNs6AWtYd@Mj>JB zZJg{dMqL^i!^Bbjm#Cf%^s8!Jxp#^1t6&RdNVz+JD^u_*6zN*p1_06dV4-N0<64qs zzpovq(zeK;zN13n1{?LgQK;{#)elrC#*1?i_eUCpyo$`GN#H=_!3Ye8<;)VUx1eAM~GBb&kb7t%2@y~2iKZ_DDBLz8YRY4Wdjhpn?e)*ghc<0Xk;>&uVMbn)e_)J64i8 z9DyzzMR+&LJg$PH+&^f(-%)0x48op&ko`!Ep>MMPgeLuf%~6CGP5mXjW`sO-Lt@?A z$7=?-Q#(p(#7pfRm6X(E;{rrG_9KhgE$`2la`hK^4KJX}&afBoKw>s?&CS*{cq3Bj zvzQan`*Te9iAOU0CskTwH9FbQqJDP9_f9GaJc!`%JQ&Y{N)9~OSL(gGn?YE(iE#gEV&qE$&!eNCFV4{NRTC$Ag{3GGQO0nb;*+Iyex52geA_8 z0xY?lWVErRE=S%b-JDW2!i+2V1(XR z@w(UbOd%=0HbiPw(w&f3i%4YF(n>KB$>%a`(WtVU^cufuPmdp(|b z+@K#T@gTbKHld3_rh}{J6%uJ3(RdKi8&R{YHpd`TAaoOcPwR{uoXZr(mPJk;Z2-dv z2{ZeFH6~UaGy8E#t0p<1CfUCxSzD7#)aV1{GIt`g8iWy*Oe6`C02`|(xG zpaqD($DreRhBAmy7$(~K080SjhcrCvrtX-50A%Jp%urgNfefXHy)zIw5j^uSq1x+{ zthh!oni?+kJ!0y6l=Y#9aS`>;_1eNi3gJZHaRjAZ|;h=`f4ZK^8rSyuzaA`O;Z!1Qv;nd5OQv)i9dnQ8iW*8^OGK z#=4(T&$Xj@^_7`!YhyBFZ4pY9>2VJxVtPEUUew|OSCTixgdVL3rJwV!nm?OJ?_T3l%-Yimkjdej>uj5HZO!6?octa)y8Sw_`Wf%Px zU&>W82|u>4wPUvRvmMhCRA!#z)SrHVKx!_(z}rN|Xk2P1!S5Y@LRIhLX`MH@|dHI?#5#Q|Y13A*vq@qcT$r1vQ3uB!4EQD053>ut`x_ZNOqZIe}p>K z&N(L8yZ)_7wu4WYV(ml-b1lqs2g!Uerudt)Ow@OEChEJGE^6l*HJJwP6VOKmpDN#F znt}r~WTVH_pn`vjyi+^48#iDei|mNM0YG%bSgqL0*Aeeglia-~xm!(gdQEbg(-E(t zBxX2W?My?*j2g8M>G+nZ`#P!nnbdu2)c#EU4yo*j56Ct}znjmV&Rz=3$J}o-E&DD_ z)Ke;7NBjdTzzq)vv4U1_ctAay9WFOKc=G4F;o)b1mfn01u~?9L{@(mxF%G}51gQQW zkvgQNy}kJXxqI{92-QA>ym5Q;L#4joO?`i`KJ?};qV4a^ox1bscDOhH6O~ed57Q#| zE<+sVQQ*C)27e*1bX8?#cd?{B4vwt~~ZH2DD-9)iHu0SV6rfSX) z)bICyU}T{8xMg|*YLhyu#+ua)P?7OdXZ>^^;A!tJ@PX3ru?eZ8sR+GT0MFid!o%XC z5~_op<64rb27Uz%Jkb1n`@%Anglr#ES)`XR5xgs#IEoK%T!$2#7T z^N}^#l`-d|h}iiNVt-b|>ykM~`y}OG1U4*(yD@u8XB7%HS%(|B6;O(d2qMDl5J)xa zf*=H~e@YN1gcx0eYRTvmJ&ZPY8J!qpbQF1o(J{VsQW-E>R^B`~`yxM~Rk{Ii)h$yJ z}Ug{YR%6AOx-%nhMmi}FN&UKss(LG=3*YsL-_;7cbphkAl=isRXM z3wT5=;)uCK3soxbTq2N~Yc8PxsJ(NE7Vlib;U`emeD%)!P7=lrVjZAyFrH-0d=F!c zn|*$eG4;qRj2Xh0PU<8w1{d>q#tFWgle|M(-b)f5V|%bR_mT_+Xfj(YWdUZd&wQ;J z$6?6C(TRo@Fpab-#IQb^I$2mUoVmf04cdsphR7%ENb#kUIvMPkRt7gvdHSdtbu@Yr z02@JISD9(SSDg7AM45!|63K5Zk=>jzI1<3rDLKabr-jD*vy517sO6m2e>D`^$ljVuqJs{P4Y}<7(Wi#I22#XCY;kJb{2=2W+;AU_FabJo>KXS;#;tS z6-FJthC0{_WtjtfkpMk!li|seVXat7^>N)<90F_wnCk2tTCpss6}mEOWIyZ8>a2&Q zEz>%uR_Ib-GOVYD=jG6ZWsWACV>IFH8g(u;VLT9~CYP$;P@quCaj_+TvU_1uqJtdqX|188#Upwyqa)fR{Hh@*>}+dPpN#G z(8LO^H=6JbHQ}NfSR*G!xNua(Wg#V)1aRu|97=FyNC`|m7*DlD@=jIxB!5*7$zSP6 z{t6@c%WKqfN^>V*Ny%Rwmi*No$zRI6J0q`0^7vMyzRXamj0Ik}GPG*Equ7 z4JquoZz$Mv2U2tO+$h_eO`u88?tLcv``n6|N_6Faww{Bu~haq3g&y z7(ivi(jk25qVBBWW)4zxC_ivuco<&PUHmi)Z{*T!JgvJONv-tEC7h2UB8c*A!G5hauGL)dsfTLR!$R9U1F{8APFzIt zy6AVD|98eI(L6mv7$TFFNBo8q=$wSIw3d%*D=OY2Gd6$JV=b?z1m+{JXzj^->7pJ2 zW$eEmqlV-h$!I#!F0jZ_kT%I8-I@njzHaRl())OgbxZpe=^pcSXq0uFLz`0pt;v3z z5^>#nPY8b(FaIbEwyBSEYDXiHlB;HjW!&Olxf z&6#}ZqMm?gns}S7rVEwkckDea#GZwMBz7U5|2VP#2@$&p)soo%hKc=ekl4kfkyolL z;Y%0wH1+Tb-*zb8idoP^U+S9Z1OpAQROE6h&r*@OD03<5loPhaT%V0!?j(X{`y6~w z&#<+ei#IZK8J@*;AbE_Pze_m})#xtde7s2)asgjD;iGkH4bS#@2KE=GanxP%oHoj? znV@dNFbYUsNF>`})-EE*^7PF$>J~lsyo3dpbhWQV`}vqF?S94v=NkEC0ti}Qy*gR( zWqz*su4Qmwbv;qt6ZQ7*-CG)eRWM$^a^14h`rUzw1;bwzFJq8|KIs>0FqL_+W@;&p z&(oCP_`DBQk*rx4thuDS{c?~s!Vr?EQ?mTYMkvXU zNy|dhZ)_26eqOZKF$NNq;;C)p+u(+z; ze3uVn<;(CB8!7Q53Mo% z3Pg@eFV!nG+|cHcbia#BCdM*aP{cVT3WwQT5{j0wP|0cklXWu-L^1VUP% zvN&mthqP8eT7T9g|1i>;V3kG-wXHvLNNbxWyEH0ot1Jp>$uW-B$nNGNcyOL0t@Fr& zKYhMT3T(S{NtDMtLSW4hSXs0k&8t(?)4a-~tv9dosIPggrv+xd0@U#k?WMYaq#T5@lG+R8K~kWxI7#gbNv(vWdKoe8$NRA=qcFz$l$gN%SiK+) zvt}Tty?AowIqhYCbPO%=KGDL_G2zRiDx)x!`W7n;*kN;9;&ODBhKAPB5+92$yd|Dq z3vxcd<6gz@;&^f>cybeXvbN#LEVT~KbC%KLx>4DW*RW1hRiWA=tlOLp7E=B}dia)j zW#$L=@N0WCT4UM|MUI5j@GK6)e*;6*ToYa1_#LMXsCis6V$eO#ibpwM)v3-H`5De3 z@(fsyw`!vtu(rKKCWE75n$rEFYCsg#c?YfiqjnsWW6)YFk3`C1sx)#Z6F`=HP6M8;#|E_B7ccS9>@%H^c~2I zOAd`D*K>%ujOcDCYOgc3MeQ?}v((0-`t2xn1@TFzqN*W^D!rS-<{M5Z4erwN{fsaHXD5-Up@$_bBU*d|wTy

w9~kFB&QndSc=V!r5GDk+c232b;92Nq}WGE z86(mcke~n|E7JCQ7AFEeJ&X0-?Iws*{3@7?$5sMQ#eYKqvZ(07ttdb?dsqQ#(z{RrcF_u;bFK~rP*8vky4%wn1%R(- zTyjb@IoVYJom5BxCg)OsV6j%jkj?HwUuhXDu{ zZbkmt>|yy2LioRtlFu6XHv`yHAPUEg=*0aGPEV)mOxLQTD_I??vk_EhsJj_SaT zF)q1#G`X9rIy$M4>g<+Fb%MoO5nwjEM|H}xs*_D`Rds?{+gx)l)yV?;8>$m5+=}XC zvuo9{cc-_@bRHY0_o4#qr4>MDT^$NQPdOH1W;hCfB^X(v6oGR!nH2SEUt?Bm$2=9f(->UY@XhqH0i`yIn#`Dk|#&c47Hu0ZbEdKO)!#91TgFk%J85=j?rGms)Bhrl|0)K@qoD_Vf#dN}rwPAWTRNs?%&WEOa|rw#FYxxxI*tNxUSE7{ z)P4~Rc%J^e&)Hi%A5A_dJh0yBN#ZjbbuK9x-O{+U|BSNR>X!6(f`sWynr+HXB7g~i z@P)X|?;B8=d9XJpOlJITKj&>K^E(Y)(%*WwozP*Z?i;}U1P>1|F%b`=iV0H$n!Y

8r~>|*fCb>w>U`k1=y7@-*L2nu$VMz**9XPY(kj;nBm>1KbS#-1aCdT4DY%S{OKS5fB5m&5zn2J8XNG^8e7-|GKQ^UWIF^{kCtQUvD)8!@4`8ypg#ia|qNV z3j+aNA)^$@4@G;gh5-alrnkQS}+!X!ZC@N*J5m>-Gsu%@D5Eu7}myPvPPeyXPhig_w{f>lbT( z`$Sj_M085j{?ZZAdlb>;S1 zE*;h8enAUP@eO;3B&1VZ-uNPR7KV+b1{h$rHkM%dH+0a$gIt;Uh7Njs33$T%NH=k>l)onjH^<+&EYNo0#7o{w4EsuTZy<@r|#Ne2Zq>E<$r+RdMU(}_4hB$UW)p!uhHZ$#yad`|4KtP9DR1l*zs1RpY=yJNP?jc44A0QtdF%(35*D&N>@Le ziBp)d*Z3yu{fPYw#)3bSwu^WL2N$|8>SN6O#(J2{)0V=q5r1T3BQEhb?LVQV2)I}n z{|l=lcra3Tp%Xa4uNY}9l)#CF@$_%9D53s@1aYHmh``bqdQTk%l7qlN>gpx1GzNXv z!5;wv%P=|-0z1UK1oEsXQr3Z|7zYU~i;=)7R(Z@it*14sC+y{4{Yx@OvyTGv7o*dH z%Nw(d#ry(}EgFD+a;nuaW^LcgE{`!0`EJF4m%uACw{loc%VVU(?X6)cd5F*vEDO02 zWSZ&(5+scUhE6e65kth1>f)rVVN$xpR9B?zW~~Wov-K?ub^bCDv6?;D+Jf!v zm_*nCY*9Mg%Lw>Xc5W8#hB|@|3o!-4e>~<3Lv?xg;+2~Z9&H65Q0VqLGFiB>yXx-H z+{4h^J*Iks<~>B7y<+y7Qm@uyN6ftfVn&RY=h*w-if#Y00yxPt~SVF;to=rqTvf$|ckhF*EPM zNgjXU-xy5M(mN9Tj>Uqn1o+Kkanf;l1x=BSlv7()dt!N!ov<)hnAQf3# z#&~UPN#;xpRTxh)0c!ja_tOO9NsOxnW7KS(z`zni*$W22@Zf)hFMu^W6XTtW#hBur z2#mMY7>^HNylopWcJs7`@h-%8mtruU3|$!tj8BQ#=R!djCBe(OlOZ>_d6W@}=~mG3nCWNeEKXVFB7L7pyLy7ec$`{0NTD z?*$AmLI$hTg~a=&)^nwVM< zGoMO(;QM%WP0YTU2EBkTB~sVMFg4>&F1lBErW{`CaG$18&_KYsoXbk6(c`7oC0K)Q?)j`yzEXwFyr&e_+Ou`(#)P_Z z+wUxfzEZ6i=4>f=-?McUwz7gDT#)J8sS3Bpra{4rQfj&io2*az+IKqY@CEhy4$*7) z5S#2fVgWbw-85*M;%{_z%5oPh&=@+l8sXTw8xtCJkJAg>YkGluV(LER+T6OoUCx9p ztz|t7aumB3AR5>**m7xL%h2}OUuXidsu3E6i0OnMVS@EQtW5>{nY3U+P_dAXo(qfl zP1b|n?&6`CdN^i2neO7DnEfElfc*gfPoy3pBe$JcI?;MGW-ZQ1!4m1Q7Nf^n!X8VC z>k(5+iCw zoy$~z*ZFA7?H8NV4?}j77?`^gf^r|jq`Jx4$f*)cStZf#uVMjX6{~XRTW-c+MeQGYX3QTdVq@BrZ!p6#H?md#+9;wa6QmkNt%Jy z^;{;JmVP#-o&&q4qX6`=cg2R1qMnV}&!Cd)YpqH7p~O%+@V7Uhv2i$_x17E#Yicjst?pjYy`}vtGxtLG zq^&cU*H#!v1!0U$*hl znKGz{wXsLtz7^JUzr(j=J zR2+t;^*EZAdgK3!__qVZ<6Z>yrXl`~n0gE7&GHjJ8zbb0W9kv49O1S9It>3v1qNlO zYU^m+X(;G5yhT)<-exH@esFNphI=n9N}ANn+g;ewhi@H=$JN zBOYVo6c%DE$MM$p%!cAmqBzb6LGdS%!_>)E!oK?=x@_UV!zBcnH4)fOC9UraSpKun`e0BSpHFg4;(5VhE()hh;;J+b zl;+_F0MXR&{0rnzC2{*N*n#Jh*21hsE#I z>lc#rdMricxva;m4USD{=pX>+6A~bvOhm&+wG{*yKB_I|KUatQ3uVIyHyEM;b%^6Y z+VEjR1uOB-bl?d#+=JCRjU7qJ9@H9aU|xHZ)KTfxW zB^NFcL2vF!>~1QYzo+lB|~;tml5KWnAL+VX0h~| zOQqLbjBalUJ4S12C{)_1B?%;0y=+ zpMD<$vCP#dA$=hCDq{%_-C$32gGk{A%c`>0q4ZxF#4*#GgXr~rWl&{iGG|?u3Z;tl zr!W!He`OHIOuYb$Tb2o{x*)7TY7SVL>j|jFV|oFc`=G6;|JzD_JJ_di7f|6kcc{!< z0EI(-DBF(}zzEM5aXp^-y^j7pVZ4e#z<_=;Ao0>K6*s(&#Z`j5zJ*oP#;v`4wJhVf zaLnw=%rZ6svsFYkJX`(ac5NJKY>FjO*$c-CZU-E@e;jS0VSuO^w?Q*Ob~zu@gGiTT zqG;=wnLWh=0Pw$E!@h;~(i-Fw!ut>r=F~V@Y!6By*mZNCvy~5K)Cgn?*Gz=8}-H)2dzX$MC>i@c#*5_!Z zgN*8^xEf3*JqXOCK_-38lqZ0)WhUdHX8I6H2wNZJOD8oDY!#C(BPShV=Jk>W!dt!L zCaX|*YnvqjEyM)JM5@VtbU}THCW!l3V0RpXh6D(Lswhzyg_reoJL9Ht|E&F=wWqo* z%F(&Q0wQB;A8JIlURuYZUG-MfGe9$A~=7Q4?kii-q|NAPh;p(p1uks(V zZhLf`0i+DtjHbqj_4$b3LF6ZRX1yGnlI~CONxIh0`BJXN#2N7Wo@W=$^2WOd%qa6V zWIbZ=q5+ZnL%vP*n@ROwnCia{SO1Mu{kOXM%~-vrpm-!=sYa>)J5&FU;rf4a>is`K~b_8u3x3>YA+7Z-N=&{sxLd2SY2e91;c6*B0z;fIo`9#+p1=u9)?Q?V3 z=z5b>(u0{%OD{YFoa&8F(qC5crITvnFr-MK=RqlvY8Gtz7;O56vFYbvvyR558Q8!X zIx()cBl9DsM2W=kZ3>~vQ8U%}tYA2bG?EE`L6`ASUmQUZPfl`7V=X@0J5ujF@zs+H z`bGk&GMM)HMi^f`34;4O@zs+#@Xi^2yenF=1g1`besXO>77fh7bPMBC(DmXg52izW z_0GatwJVqi&h`%y*<`)zLGe;2j`UL44{I7RxU9lA@tf|{y<5(@cgt1xbY1rV)QvDB zYY*~N?+^@cGIYa>^kNbA7kcvtb`%l}qu39`oPgH^uZ1~*7GXVqbBP6|LByOoBL=+x zT<;@jPluqrd;~c`zI){&XirU$6=Wv8X)_9HOdxgjYRrth8Z*-&avvX&JY<|ovQI7| zXKEtrauK;-K_UaGtCz_A@)EhfL*xNIB4w@*A`i$#9pwte@MbN>Tpg|x=9g6bO;~!>v{6phv7VT?2$(@bOyE(jR zc{U>h=5ps)XY{gX!|2IE>#MT%@-o(5Ud7rA7={-3h2Dr*aNmMQ%;CG7 z!`?=Z>Mw}tHs6+7JT_ME3V1oYD2@oxcgqw?u^El zxy+tf4|%h_>3rE;=^eEOrk_Uu-ph4h1?sb>amegU9ESpsNcZTru-f(l(!GGA9Y+VW z14yn9*i$>YldqkN?nyTSK=h>V_qC68W<8&AiSZbxPu-XqxKH315ZwAsg@Ti9T?J+y zpA6*wjlgz*2 z*Og>);gHcdi~tVS32`odXdtXr&qtwFfe;E`Ca#eD#4QK@Oir*&W+Wk_Tb)zLc1d4o zpWyhj#CUlM;2ds%Z?C{v-Js* z3j+u+Qf!pbS-b~E4D)H0p=u{y0rrOiO)t(lvc7dsO^VK5@4_kCI zC@h)N4X$T|aXll9>*>H1f*A)?Db6#ckiJLLi1SQ;Ar7FoKyJ@5&&L&QLniL z7CAN)>r~^CXE`=>JF*f3%OwTJT!GZkm=@K)xba@%cSp$N($F(xLKU?*bLmPKI?z2QaSHT;XhI|FEw zwLOBVIN@lfnw9i5(??lnL$WC1? z$Ofb^4M4?lIGr!$>TGnf>KwFwYe#RGTlhZCf+`tU`-CfeRfTm;#J6N58{cT$}Pcswn0d0+cd*nNkT8kMs{C4#WFN;s@i2X5|_AzX2 zMR+OCLw(lc*hkk!|~)p@#KT?Ha8u!n)g~jn*FC?R@zI$Egzx8`e}Gb*SJU zpl7-@wY!H}w^zQ?8DHIr@s%~ppnHRiz5IJ`gdkwM2Nrzd*m|QGp>yWhe6wfzy|{WG z+OQXjK+x{yoA=`OyJ#4)`KADzSn306z}^!}XJD6PS1GfCyMnG`>veC`(Mskhk|V^_ zhb%i2fBU3~!@j&XV?W%P@nIYn;O>``8+SY0xEtJ%>aOEech|{Pcl$SY1|anTyfQcv zgrjvi%;XJm&m9z;(tX6*560gi*2i%$HW%~#46a|k*)LYfF#ZoVAF4iqMGHdx9N(;F z&kV)>IL-p~bgyB$adC2aX8`Zo9Sok?u*xVV5z@Hjp`$=4V zX2vtK;fnCO`@W8=Z{p^Yarb>4x4#m1p9lFwY85$lICiqAZ*x%oLy&S%YVDMht+&d4 zq!pN=+)6P@RieGnoSY>77$A}Rh<=EZQUrv4XGr`uuD*vF9SJS?CazYAZ*F4g(U?1a z1Py@pxo3yHhHS^ch~inC5zS;6^?(bH&{O^#PyS@gr`$nh|Ln2h^zQst(7O{z4S9Fu zAg`a$CjZVt^KZu8gQjwc|M;KZAvIZy_ya0|31(oS`48U`!>`)s{=4>f^M9TBs&)*GLX7@iR_%0$O?zZEi2Zh^){=a^IJa zNStL8vE3GRwX5`&fgxQI2Ky1V79yIZch*VJ`$bC}gV(XQ24ZUeJ=peo;L ztgo{`hcafiwP!E8d%~>6J}7Ik_hT*gL9E4k%jP~a&BN&hF}rKRJ0fenv_~1+vV!Yr zpl!+SFk)d>gdT@ z0a>94_rpyscUMk-ExK#q-YawsxAnExPAqRcnO?rLImWI`m_W_5n1Sm>mB6sol~bSK z1EU080Y(WY=UO-I2QFpTP5UNfP}EN==EA}_7}Ti=-r=`*@o1e~(AUw>7quC>XUOkc zZ(TPR^mR4##chVp>5DV!^x|HX3wo7?zC@sZ?vak|SO^;y!o!)7fc{nz_Tt%EAMTM{ z&?6f9Qb5m6>iSY=WeZxF!h3d1(q7kcH30Fl^DV3F{!qHNc}m!iNhn<+k&Gvju|zVO zNY;n~<~S$+&br_^`5Fim>uuStn(vUV8Yr;4Dvc3EuAU5AB7?Y!uf)uXzU<0OS2#xOI7&Dd<;sE#o!?m`bul1W$oY-4w6Z0* z998h3u3*-}z;S4tvd|F3OxGDRZ6(w|XlhI;CuNXHQ3*8|DRCYUkgdp-lP|154)3o| zK)h|b3qW%o&D3}Jzf(SY_Za1GM$3?G&-HF=%k>yxVWi?zK;5CViZa0mgaxt!qv=8M&$e6;z}rE{7u z$&}-;sIYDI0s-f8+E&!)IV?(-U=6ytp;S)G<|zeJlP%SXsma#mE2Yg%UK^qv+e6YD zQal?bra|e7QZzWpA)=-#JLQPz3+mcNW=ysb$@iY~?qzHsP)tKGVk)%1%-_-iLpLuJ zwBGFgvIeyM^xs6s*|MR56Nv78FwM=igQH=Do@IFU)wPlCa7DT!%TrM*olqla*hb)Q zWOn@10Fyt8#%dE7t2B&Nlf4nABB_zg95_6Sna40Q&*mRV%Z8jfCXh*uP!v3~W2p99 zqlu+kLP;i7*_%2_@+FmIlu?qctuYC@$8{>X>-L#`_c(kFt17tnGgsDtpQ~}Rk`maX zmh6W3ZMY46Gfu+uZbJ{G$|T&o4PBp?HL-L%oMmq{Cb%2jQ^dC$y)E1BrFXH%WJer+ z(%d0RfHX~(6xOFBh*oFAfWn7_d-&ziCqO{gjS+1iCWsTO;F1u#L0Rj@>0M2BBbBLP zES0Glf0NYa>~Va@CFR_B>2JB(I%}!xX5ThkxRHHJ4iz9=;R23LqHA`X8S`wOP+LH8 zca*J`Td^s+o7fqo#-CeZzKVz0#6B9icar4~ZeA}}V`WY;nVZeGp}yqZxE@FqeerKx z-&OXm?@CbH7X%eZ%>inPwF78zf=aoF(&YO+jeFHS>geCA-b4os_Ns5sC-$oEV193- zfAg${-BhzPe4J!h*_=>2f;oFwJ0-YB-2*-69(A1mwNnD^mkL=DOYKYr;pty9@@^O3 zfnyEM2)G5^!_LCn#b@S)*W`ug;b~!b`%w7%L-IR|t@P5c3u!n2e+R0`|Hn$Mc`3V# zr{+-iU5$(;C)5T1edwCdG$=;$3L)A+9!d(%#lua7V1asRue~ zFBGU_wv(PEL`>I2m<`|#5z|9N%%0l8^)I0f7uQ7{1q-%)mAwbLs82b~|J)2UvEpGq zl1&`SIs1_J1lPIhfoE}5JF6bsREMMj)ZA|5z0l+Q zomG#0X)AL2%J)Ol5dYx*5M6lHV}GA}aHjT{?I%`NPBgQFL>^d>$Uy4qCGx<$L>}Z2 zd9aU2XVv52TtptEiIi23Ad!a_Br=e?dWk$VFOi2iM9%UN>8yIp%0=X1nn+po2oiaC zK_UaGtCz^b^AdT4L*$V@A{}q`k-3OGLK7*g9zi0HDoA7?b@dW?R9+&Fc8EO2N2Iga z>6lzZ9<7O#<&}T`jpdbN6TErn->@$^jtrGimQ{g{eIL4v#bRgG(Fu#RUvJxQ1Yz3ZeNHs&e@_tjQi`-y=&^cVVgO9&Q_K zOCYzSJm~HU%rV*m$!Ti^AqhnV?<$%L&GFq;)OwL)a~3J6W}bdbFNEFYSqS4Ai#pwr z+Zje~rzg~zlp9wuq@z14VV^2B`?nCD6=+o`!len%7Q&fDj~31YKnzwUR@sZ4!OCZx zPcL-F4d*ih7s3_-!&Vo<0*VzFs_7~)J^M77V+-L@@G0vqSfV@jrP_#GP#B9gU3jEx zrxMWN((Vh7&d!D2*&00_k5_<6?L?2$>UN`dPA>G$(db=N7(F9;a)xe#T#Iq%=7N5% zhJJCIq0<$U3;MEL(3ffGmjHU!8MTD%=2CpZP#N!kf2h2)7J&A^z7Z(@J+K!$iNb*z1-uuD!fcJg6F}i{2Zh6l!sCVuV|s%DC#>b;!jnYd z2#o^Xi$vjAjlwfP;lBZyGynCn8($a1fB|RpVy+W-T(Lt2q&i;@@|KtPxjXsRT=ZWheeH35T?uD~m%1cXo zKiI!xIn=PEBRknmJV_Pae-Bo>oWqa~@L;zHUZ|X`&~o zOp;tux09rbil>CIGE}j>`J#k)lDY%Gkklr;q#q}#j~7qst^fv7?+2+*_D??HkUE8x zk<_VtaY=pBOV2bQPLx(C0I)IIT%eww5{U5L~~ zEBd?60GtF4DrXNEmz*^qdDsA>yBVmOeb@jxH!+ijhZFaK!slllIzV3l_cPY$>{$bh z*UaR^v>g6fy1d6AA$>Sti}L_T;yr&WYYUPYb57d?Yg}?`E4h_LM6EeMhuCF)B|ckQ z=!fK3(yc7|=)JyKyu)b^Be<#}y^RHowy}!Eh-bfnbxpu%dn>t}#;BiVF!~=c+8%f~ zXTfe)5T|D1)Lb-9Z)=<;1E-y>@&d(L3G{BTlGh7vXi7ZC#l9X0%(c)a=UPsi zybjDn+syn)q;GJM);ZB8=UUk|d7}kDJRSvz;=RZ=d6Y&pQ#USoTwU_mI%$(jDO+Hd zDd`J*j;ouSICK*J>|^U%8owp$kfyQ1`?tcnZ!6L#)B&Rt>Wana7>&_Mz-V4wa<0bc zWe1}&Vl)qU%(Vmqdu~CT<`bv+MdM_KD5nFb1$D_&HBPTNI0=^)xLi85AVy~pqce)e z=mgEBCBSG=U2>tu=p6?m;nE@xmlhVpX(@49S`1Dd*-qmfYg zvcl#4$vMh%6{G~#r00e2=4{6+2m%P`J z#wGX+1ZQWi!OLj8a8!UGBb*>}pTlU!GTQeBzf4#+i|zwN*G$LoUL?8?XmlR|x)0YS zA9B!r5T8KIS9_95d!Oh{x<-=9%Wo?wow#SFYu64r8ywOM2cwuzOyieb;THER9{%pzO+l;|xtB@Bx*KhIw76~3X4Z}70{l-)lMUaOF9*SUCE#Oqsp173J960iRO5bzoVyp&1~ zbWr=9bq)kVONQ8rRo5@k*l*`VGk7XOYOPrIA@5$P9Hb>4R^mrY|1WaAgk#PG=7pdP{ui zkovjtUM0iuAGy;H>Gp7iUt{o>9!AvG!8hQA_agC%0}v#(A@JJ3!K)VEP{#l~tc{hu zK_0v;q}v<2cx^hKMC;k`(_h5``qN&~M3l}tIP4P%`tAaw2!yMa~LFOHuBP2&IL z)o`SzM-Y<@^c#@D&mxf-rIFbL$c$3Skq$DO5}A=2nNc~A8BG*6B{F058<4@zB9Ylj zBQplbjCL>?k8h}E8$7H=Wse3<=M5Qp+n8k-R>Fgc80p^R&bCOm8x?+Gz|}jw8Bv>v zZ@>%hMdGy+06}7#1Fx|TUc2BM>ev+zYYSzM&4bqzq}yA#c#R`oQ}GRW;k`(_W&jZI z+8TImrIK4ZsO`f#w**3G4zagl)%8oqoB>VZ|K!!aNKcO^Cj03(AcLPpA~Q=Pvn`PM zf4rRsd{ssI_Y;UAh#G=XLER+~IN=}{P*yiwQNbX(f<#Rem9lnSUCXZ6UA>9~3#fn; z3u48B1;utm#e!I{q9U+XEL-flmQ`2Z?{DUubI-Z=<|g6#fAjfd`aI8Xo_VI8nYnwp ziBSQVlLcm!1+!NUFsBGYy}+DmU4X$`AeeJ3n0)}VcYx$v!eBEA%V}`Ey}>lE|G)>& zy}G~I2H%doa7pKU;=KkJ-!T4@`wH4*!oY>QKwQ%xM2+nSuEqe@rG&v_I+ioW^%|SO zHG_C>Oo(fL;hISpxNsMU>uLyr>mYC)=q3&bpj{)L2LNb(fA2uC?l63E&#JoV8&T6<8y$yT@d~#Fn3rNVDJ_QW~l{p2w)Bl zkUUNpY@Wn&4s*SO!8B8=TsAQWG6e|cNy~`$4h!cahYH$Lgn!pU-Z4O$ zBS_Pc5rbPW19*pc@3?TJ{j;FGOBlFt7l`X$5JE?ULGr@I+n>amyR!XpUI)EcT5S|=7fO@cY(ONLkK;Z z46gIt#H0XP5AmD?p!q%^?|cCWAoV0V*(4y<)&&;41!5Usv0Mn23j!e962>naJwih- zO7+mD#RZ_cL8$1GB&`10)t%XY6i@$D>E1?F@*)A-j*OtfT_CEYMRf_NrUj^m2vzSW zs%b5vx>TrkvZ!zuh-x>BY9^>=1gLfwszaivX0(Xva-rJ8qQYGus(mb~D?l~ZP0R^^ zH3(Q=faz!0daDxWx>S@|dF!!MZw@$bPS@!yQ>U}~o1ai@By;jg!Q9Wfs8zfLBAsB7 z&Ijo=Zem`5^k5jEj>0+AkTk=_K-8{NbW0n+1y^r$G( z8{tu7 zdaLj~E~K|v7fA6Ih;*?N$(*Wl+5opn6`Yc8j8VsYO(;2-OP~748C2y>3yx4yxAzRBs5?$x&3VwTNn!P_4G8 za2JT`J&Wq^pjzuD)&#)b7qC5jUE5xjxYnhj%q|n>R=Rj=!1+MBPA@ZcdYNs%ACNiu zreJ<(UDPVx0+D`Zk-i7gciqH00n#sobaWKyyK$uNBj-gj)Xb;xQ{S- zFG>wo)qf1GC8}$d8JP1-UC*Qa>e%QojO3>Rwjmiog}XphT`a0ELG?v|s#2)-i=z6X zMO0r2)n*nI?gCN$#-jQbRNn-sx(n4gQB>cwi0V6`s-_-ErRh4IZ|ZcuZNL4*s`X>s2e(xSf2BONS!?fZf>XEtNV z4soQVd613}zK%k=yLEvSZ-Ge1TcjIZWu1$sOSc?mHfw)exxV8Y-<^isgg)24&Z{7-AU4`ou ziwk#wxX!V-egm$p`zE#upq(pdu~B$ypy^Z}b1O3pZ-xKKbk)X&;q%Cy{H<_KvM#C? zZ-G#!TBy}P?b$ccBY=8|pvH#bo*7VkWeKMfdKs||_Lg38YPkZ357DWU z;vF#bW#wtYcxZdiA@0Z_Jc??Wt`Kbt$sR-*Az3`HqxS3sCe7XPG+UGA1JXq008?9I_r=pzk@jCoySjg|9`Z8u?vJPGMH(y3K2d2t zM5^SaL#fkChZd^ePe6;U>zUN)jG>9?LA5?9Ak)KYwH4;gAmG{lNYU2VGU2#fI4&<3 z#|sw6TyV@8nwUK_F)M)bqM*zQp;*1}W)rYNP^@CS6i;Lo!<#cyN)c+t%ffqw@Lo|c z-qjZG)!@BqXyVEM>>A;?G6ZWC%e#tzmj%{p)>`41Cmiz%#_^uTF&`Y)1R&N4#5EZp zjCQAf;V;DTzHnSC9M=|%Z}Edu+P7pwWCH zh&Ku1O$9^zkA-*(5N{ruSP-E6Mkp7AC~be?-AurwGDKxNjBkbGR^hm{U>rYN9JhmG z;n2iw0ghjU3xNB(Q&HzUzg1tLK z9JZhG?&84kYH;`+)keZ`w{YBDFpkYEj>X`(CjhazK-?38$ges&*PX38TL|L4f_QJi z5W8E5_XF|10HPy^_Zf&9QfkER0kM0aV0N=CyVr)Av#=^%>bA(y)2^|jr(gHi1(vV- zCtnxy*9&1@zbgfKu7gpFn@01gWOU+&(f&Zcf?#4_z;bUH-N0)`sTXlOH;?vi7|pX# z{RhrH_doHIGA|LY=N3r%vd#U9a^E`InY+98QaYzj)iV6%(YVbD+_=0Owgd#qHcR+FRI+5Z)u+7s5K5`+eoUJ3|O}g+eec?{@h&GD5JFeIc0Gs=b&u~mZ~9PK`&0)GcB&x9Ev z+^?|j#j%}>yFelQ0wIL(FhW=oNT8_9NT3~-yVOdcJ#OdG(cY3~Bv3-UFM$ng?o#D` zEJFg1h7vHYL6pC^oe_W~>gu!Mav zn8>QZ%SXGVn146GDh#1!8z^bmG}S64eC5rZ<-RqLqY$ z+j(KM_iQtgs3G2$L?4^mRqmHEB=JHhiRVI>CF#q+B&Ztv;%N6}m2?}3cqYu0{&|J< z6Dz(V66141j>`Q; zM(JM<%WPcUO8ISju#|lvnAobpt46!4Rp6bV<(V)egx3_dvpDh_T?Gr_00<$3Hxa_$ zM<>>fPOJ&!a3C?x<-6-MU+2{68ruF&@%_E2^aj=1T!Cu{G`m*eewE35Mz@#x+*(2qr?LPQAQY0sjL6DQj z6pG;A7QqD|m@+2O6oB|xAZGd?{w5Gp0P(CqJlo%E$^i6<09`0R7ZwcY8w+R}fToT~ zTs$UmQGn@NVVdh>I!BnMj?vO`KN;S|U|J!3EBbpEW#Icx_%0E?OA5yKv&DBA_%02Q z{UT&HMyu@73^Aea{*aE0LhD1vzz2>J@ad?A=$ z2!dC~&_~rBVYK9R5WHp(c(09NQBQ|vZuyoK^@s#mr&;6t@=H;X{hjopq7FqHu@P`` zs;H$LPI@WI0KnZ<*i z;khVUUqf=$m#%0Am9~pO{nvuRT_C8UK1PX#18Ua*R54+3P`l=-wB3ZK9a&H!+y&z4 zWby0?o;?CQ8wt;o(MsDRPo<3%o{cRY+y&yc>(; z={eHqT-E3|yM2YQn?;DbK!iOk!UI7#W^7{r0AbH8gkwM$IqkE5o;p292&*kZ+yx>` zScHdw@Q-5?69R<&g>Z#mt(QrLe;gY-B`_gREgvd`11v(^1tJ`55&j8;M~zJ!86Zpw z;bqZUJ}O?zN9L*JV}x)=ix78#2w*u?Px!r@s6PY4k*d*M*m@p&NrMG$wh z5OEg>aW4z;G$5WbHgR$QadZ~MQ{swvavnuIT?p$fLfi!+9AgotKzR1p#NPsh2V@~U zJ49%O__sU|&lSW2EkxV}LOjeuya0$zV-u4Dh=&W}tI@X76mKh&LD<|dcA*d+VG-gk z5aDqa;bkDaWNczufbjS%gqMT}%goNlv^)@}3*reDBJKhqo?#(g0mM0D6SK!AW(6qE z6w25NcTU(|OmmqX_s#-ybF-%_h50Ot8Fzu0CtJ)nfcd(yiEGCu<_DOYvM^s4s;9wx zZQMIQkFwq<%u_68+y!EuVKLtU=7nPuw~bBQ8epC&%&|@Kg`qYWWw@$a9h+Dj0G=nnH$@xRy>Wz#^B}xm2(PgSaTkbifkpT*2$zgaJQN_j zSqSflB3u$j_)s2%j|kx{79s8e5iYU_p9J9(V-t@D2=5lc`=SV+h$DPF55i?ac#lPh zd;Jm4IUYhMB+az;Fy5LPkvJ4HaR`PMNuC@{!)7q$=bNpkxX`C{+Kd6LjWCmk6T_5N z)~|%-PmmgsI2to?6ox?~2X|Su0QpZ@k*CNIPB##G@dU$3H%kaHlgE%4gtC4mgeO9Z z2?l23cnpJ3ju4WkxX{-rDzCo~0~%Ss5}H#WH6n2`W?~|SK_f>*UWyFiY$18Y6U5|o zDv3cT>sLZ}7NkZb&crm}iks%z@6~q(7p<_p5j8Ex`GgmM+|6W{Yq$-LTW_f5zNHH7zT|Tk*ZQ;2*-2< z@dUA|dW^&%locRs_udpd*=F}tN!M$O>Z?*e7u65i_{c6>m-T7sNv_K;hEnlg_$Dw1 z*NDVy%)~4V;jS+}4LXxHEiNiI`@4E*dZ?Za^^`3Akf+isn9l`YmW^U?=1I}Hgz>XU z^@EB`eoA0M(({^bZA5w%k9}_j&UZR{8&5AT($8=5LlgXZ@`ywy{D}=Q1lk#7!J6N~ z(=SOJnd7(c0t)=5!P4IP`Mb6J{^^3ce^pneKB-_QMikl4(%XuZe06n+K0H)=;t}~( z=B849FWLEBoB8o`a{sC>g6n71Mfz4(nfeT-qsAF}_SZv$UzL3z7kpKQ&dN^=C1K0q zeBGH&G5lTT`r^&qauEB!nbH}j`JINr3Un(l>WjBtMq=Hk zW8$|<|RF>R=2+AreQnK}F6+I{%1=tcwe&Lo;IF&0} zEiGtLkUOU)nc8cWqQ}WJtLXM8XnQ8NvOWha`@O9jIl$q>hf`$Rk zxC#R|0&pL7<~c=(Ci-C5O4!{~K)Mo~t-Cr2LQs}l`>0o3+(&)!t$iv}+wy?sD96bW zpZf>2#AN+`)=mJZv7A3_*Pl55r|kyd6j6QgUgL|ypKp}V3hV*) zMdkk4*{g-pg1Uz_L3mc}VF&zH`H1H?k>qd1uOj7Z>;VfJ`@_9+{3=os7SZ2D8jRm) zgd(I7jiz$U!^eu_7j$WL#+7b$Z5<7EG=wTCWcvzlD4qp%Yb1ng1|Wo;@h660ND?__ zNw4*u-XlI#2NWpdnjEtvGj@klqz-n9BnGXRBDKV<*^xt2uIwpN7C9-rog&2x;dHBr zPLZ6c<)NIEr5971Px~P;K9ti3ozD-cP4?SSk=l{7sN9nrnRwQa>Uy=dT~Dq|T~8vE zVAqpJ4V+Rst%ox$7g@@b^T*$nL$9T7W?E@v78Na3#w;pcs^~0gbLWm+%t3pt=^HNT zQJK0HxM4Zy_CBt$xB;<6av0@96=jWY$0RIZHB4GiK>L((qM2J&cy+jZ&HSgD=dSpT zQkJCS5Z0RB8V%m2do;cM+v*)hdP4=X-d6Hu0j2-`ZHv3-{&>SYz)NF%~k>bu_aMOa^49R>#ggJ0ntMe_P$ z{T`2Tr905|1K|RqyuzzDAWR4Pfz)&_{B{)S>&cy-!Z43C1ET58_cmY>fHAH@!E6VZ ziVCkW4dz3E**63;$lV{eKQ@oSX~$*-x(8s?7yk}o>_0gXSiAer3a5&8G>in`YxMjF3>hvSSEF!=Rih371w@~VJ4 zJ4fJZ%Pe*mGc7#QFicNDHIvQivYgW@JdyWjvd^2*Y6TvBRg3O>RzkPQAuG@EFb#k>Ag_c zE(0r56;jOrM!JVKmF5K{%+||4HwR1+0=7qbqAbv?rRckiYyJDK;|-zTQafkh3Wn&_ z)g4&|udYtc6Kg&vC{>|Ms$g;!guH7o7Svrk0GZv&d|dS(hDlud4bSJRhj9wMIyv70 zkX0}Mjb;}Jz_n=rcliL)qtC6~7u&L?1Dra)8#4xy^N{a+#F{5Usa*R`sijhtRgK=$PU=z2E!QpLT zU=z2E!Jpg0z$VVmpuX6=zY=_H_^jWN8EVU~mxw5X;10p zYX+r^1#BiIt_u8<&rANN0LQ(PHtWx6@4{*4v+y{AP$)k&Z*j zm-baZ@MK@VAE?{RY!)bSdb8kOeDTeKfOvefKg4#)n(dz83)(M zc0rIQ-TKIvbBC+>e0H2^MLzofC_g(`kS}-tpT?PyuZ3>C*?cYZ*3IT?p%YfV`r_K; z16HwQO`sV?lxuw&PJQuCDAya^yob`t^{m>?gZS;bV`oq9^!mKMc$hPgcWCcmsh=y2Z+oQ6=T3%E8D2X{susi>IxJ%_}Xzb8f>OLkSTEh-jp!+z6 zK9q>IGrRn#vMsh0jFg!JE~uN__s9&@4@#$05q@;^%h}IV4*1{S(_nCj_qz-*M_2oTrp! zdR)5UoNl^-aw$-zYsJs7a;e6iy_3tymWY;WRSMS7wuuQQ`{w5w{L zAqUm4FP|S8oE^$T3JB_AZD@bNfdmcbm8mauA|c#AXxF{=S(|k~vX60|!{cnSbyof< z+i1vZ+NLr!MUtj4kEt*RI4>x}ty_4?s6cp4=QE!cJFFb_nUA!$>t(b9oh##|3H9*G zU~A+`Z3wUvA60-bReVq0IcXD5yu}EfPpEv_uzy0uvvX#69@OBxEP#!%#EX=;?oI$M zQfdI+sd>)Wz3>xkUrGVH4Uky&xjGm z&3s18Kmd-&H^RdjH>xi_ys=Yq72I6|q2Z{K!Q~-38rN&pL6iKd+U^>B71J~Jsot>I zj{y6T?rM4X*`h5p3Jp!QL#dqKrzujSUz$p4Vv_ZxDDBr&)KukM)XCXCmg;kp>O)d} zu2kfkVz+*t0eA=LI$g5$2ms4qS$y>wn;0*-Fn zq`vsijT^fvK;uS@om|PGal-~r9hz#V_Dvk2G{WHSCY<6P$N%w3t5+vTn16+c@<7CS zE7qH@&^$+*E+dghND|O9dZZgI!hDO|P(6wNiJ5>;7K>A0p+t=vHg+_uU8!VaSz`wd ze>IjS-^S;frr)OiqNIE6JC^n7us$SyGP4ZtT0&<*=x>>2I9p{nhcYOU z%D~xoT{&(nOTH)M21J{XckEuf&N57$3n7@MCSg186X<+|k=5rtJV4#Vk^z;ehq=V3 zZepeSywv&1_D^Lq+dm?pD0GkQAHV0j)z4vizSLwfbUs!VdLm`@yRBw=qEr(ZlAn;y z{S*V9Q^ZpQqCJulcnS=9q~ropYTGi)!BqdfT~Xj}uy_ErJf*7a6j`DMA(FUw`v z?$-*;?zIBS`FpKMs#f+|-@-25YgIOG>i&mN+iMAq`wa$#o({guMwFT%NpvB0mK+tX z3T1je$)<6W{H-^@o854~Be&{zHdJNL$;DegF>b0uKQc~dt4xiXG;W-fM!V8R^`+$2 z&yF{4nEYPBK?}W_Fj1a|?f#b-W|kW_Zsh)e)0SOyOVUujB`ns&MVWGQtMVXd5Qm_5htwLg(vxE7F8_RZMt{}h;^6=t9^YVXf- z*>umf-V3KX;a+Deb)#rTrf$%+yFUZP%15BwUob#<3n;U6<5mJ2ONrHwjyG=Ls%VWR zN>*&Xc53izY~R?96bn^KGZ?P_p80VW$|6W1t2N!c_7uQhADbmiF@C#*Ni8OkvjP5m z2lg_gneo*mpl-#A3Ts})!B+L(_X&e1`UIHMuH9=(@u@=ZR~+_(*lAgm5KEv(pi!nJ za+J8r)Y!hkc`vf#tW!g|_O#=G=qa9J1rGnp)Kgl4`!G8AFdp_{EX@VuF>#kp>xbBZ zp-jLCO7v^Msj4mWkvs_x2}HlTo+Xf{10>H{B+4Wts!J3pTxAq@ukGm5d(Nk~A{V{q z19~r6ddegLZ-niz&I*L6h83<0nptP1UWP_;V{*EiU{Hit2r#5Hh6d{`CGbQ)SZ}q4 zM^$Y(IV6^q(8PLGq&r7RnS~cq26M_T^&0U?{<_k!JX2D|p$E;a%u2cs?n0_orY`MX zTS019S;T0S&+rW~+>}7m7u9w!=lu26CF>JLh{^W%KsURfsnw!na+@9CTR0&e*Y7^@ zo#a4%w&e{(#ZgF45JWLc@q5;8=0jfNLtcxWrD=aBuyI2Q_NE*iU1hS9+06h~cCe@7nF6~_+T+^%z?$SPl6kXciyLnxq?{{g>X;)@*{C=1A z5AoX)KPmT*E_zx$Pz1!=I#B!urtwJx4}ZE_5h@`U4HhVOYYfEoZ%{fPRyn^_o&|M} zB((MVw7Eg?NP?bfb9yT0^BnQ!6V20#9;a;D`$~N*Vqe?Dka-R=O?A&e<~h=ZvXW+L zFKxaS2z1f|n@Up~v2c4hAsV0gfV#H;C@Ve}pwA)J;Pix)`jSAN0?=Y#@&#Fw)#P-0 zVF2t)0TU^*hkc0>cnS>qlGgC3s_jh6PQZBkNmRZfDw6{m+=!AV*emhVAv(gshh7zJN|oeNt*Cnl%42JshVm8q@}Voe=x z3OFS_B0lrujNknqK=r?)J=8mMKA16s33MlH?|G1KkcIpQA$JfUYqO>*9cpjVN2%0L z3a=sDxi-g%^=r(Db?u#8qe+R)<@O=Dc~_h5a-6N4S_06QH_W%mH`%k3yE1i}bdUZA_f&;Kcg_s$qRCBks6fOBm`lOcF3*T!q^Ox9K1%p=w&&P(o~tD9#;*L1{M^(_L{p zhi7oUQj>#W6Dw0!s!35z6O9&jC4;97!MImdTZd0N*o`RJTgrC&W{7+pRid9zu1pc)!^=CEN7hQtcp^PvPsuc*0{E5lVJozY}rK4-5tYhPi_SgwjRW)v4epWtF=r> zdyo!De=t@Sqn3PKO2->|40|0?9P88UvAae}l@Xp?I+3+(rya z;07YtPtg)VJKOCAB2ajy2y`4X)t8*(Hwe$kn(naR)zJWmqCT&7^A3RTg1Y58@9~U( zAOfZXi4UVBaD6`wj{hHMi?l6iIGtg91q0f_0(!lhH!dfj-wRL>B|usn1c1g{K(!W- z@f8ecCku%0KnEK@=0ptNfkq@I;P=;$JCn#c6u)Nn3io#pGR942Y*WoVZ&#&1OzEHP zh25mX_)KO0Wjj z#n@QB=9uguCfQ5HTl!=!6-A8#a0@nTsVL_8kKL=hHZvy2?#;PPEhhdT_BUj$z*=B` zgHb`9o~uR$ouecmNm@MkG#gUiOVp1hwk?M;QlpjYPs+s?cjeCc#l7W1rzK_!jc(el zym}Zkc;cXt;M3PJ_%&qityt`R1>;x(V{@?v#X2wPBep@1{x* z&RXNKoa3P8Jl!d#!uIGEI36}Ntl8HP0Xd$uQ7Em=cS1(Kt(-TSVHhv?iR+o~FIo9s zY8Ah~u4lfJGV__$7t;SD5^nG$VrsK?BDPfOV1;5ID*0R08aA`{!P|S1EDv~ODxXGCM*^^f1 z!!FtcN1E^P=Mid`DuUxq!T{gjz;_kB#pEDXeTlir=LZhN9&SF}9Aa~vE$MKADRqt< zrLIbBHXI}`&T!sb9;@?kU;pahgyGdFn$cEj;gTA&$(!;~H2ElsqbL-Zg@PBmxIrH- zlnl0lmjO)>rJi2uHX?BWOcPTu{K@|mqC5p6jzLT|i-wQXwcDwLgmc&&Z>wa*XU$U^ zoON4!VzhqOZp8pO=6qXa8gc%u)Jt&Md6H50@NE@^o16L5TCE$S@FMi!Ns7tg5OA-U+|l7Mhk9s;V8Jma~52AO~Egr;Cgr9W&$Xw zy?>ME-2ra&@l1{09f;+KU-DpsBFDP}{@9X!cR&;<9PbV^-;v!y8o!XY;*1uu)UREI z=Pac1nnGTxLf#h7POo~B->Yt|URBYO^ng5I7LvvXb9F_g}4y)K|BF^17 zLLW20C(SA2Cpee?Z&1RThW)5v%h0slQ^ZosTmQFFLdXyEm%W6h>npbd=DL6rR z6U0pSbr0cDP+F_xxE}7s$=PX8pqHYNsh8-}0(gus4JVnJ92IeVV?KLmP?@u>xS0cTprm$ALUXfAajB^YDBhE1_@0;U2wFh7leZS&<{AL!Y2-7U@ zFnGR*;aL<+`xY_nOV{gyy48uir_-{~yw@u>ReZ2HG>R~2?lfpl%-q1s5CUzH#{lBo zXu6jNZ)#N1u|*Z<+?+@9g8WLUEx_;lad>@cZygCSPl24{lbRAd|Z=9eI9KNo+1pqdkwt* zgO-qM>nNs*+U{94fVOZ!U)sV2rd!F|7M3dK~3_`tOuEeR5+ z>ygH`^m?PjCzVDF@tdOY_fxXThI8wTnX>^Jq&c?_(y}iD8iYGJE)#Ni>gM z3u-k3n>arM-rqJOjeqb^Pl3d1CgUlP0F2ctvaYT!3!Vb`O_UPvfoD+4cnTzw5>J6t zjSimzsdAsC%KWE5p2KN32o>nAz~H$dJp~dFiRnScQy@Xzn?5ak3Z$3bz=&oK>n?s+ z)>HF~G!Op)kKUQt!$Z5^}ke4(U@;L=wnQ>98YLu zcY<*;$TMdr2-nCH8bO}4){?KKCp37{BAU;tYqO0z%`YQg3!Q>}nxbEFrL8bo6Hw1c zrM2~G3Om4UOQl`h)q6c%X_rVRU&U`17K7Y3aFeR#3lKS~H6$F4cCGv(u@m{E!uIc9!CoAAYZk4@yY8%eeRu$VZx?nhK_%v5OB$ z0eQ0pAX)J_fb7Uj0x}L49-@rH0kxUSQ$v#PlFNM$gEvPE3Gjv&9Te$_ZgA0|HMFa0 z*O7zoQajfd?~>!xO7;Z@6_75yxeE>f02a^Yjf)I@@FzQdZ$7kDmm6XnTe{pZjPG=k zkgIal7w_7tOAS(ruuKo?pv3xJYKS4tI1(Sx_Uuayb-y;u$c2V@v(4BD2y`TDzj;t+ zU1%8LypQ16>WW-u*u(jNur^#|_>kl}Lb+$=C5C^Rh_+v1h*f4ral%UsLAll7!x>1< z#QlH~sXJb&&`DQ;^Di-cgwu9fDvJ9L3|`vT1%`j&q|<8l0z+23=_}$F81`}g4HaEp zh^3BSUXaLD3+OP1<5Vf3uT_^9@`)@`c3oa*jo>0#!pjQ{p31>mZt%z0)OREQ`?;UU zlkp)u=k2?=7Tc*rfF)GkjLwFBKIiA~&tV$HXsh&7LTxcuZ4C_QS&g#>ZL#f1b>wP7le3FAV-kGR#)4>lW#KjE{| zR=E2=3_YLoGqx@w#5m@@h;X>4#I#?v;i9VAU+_WRJY{1}`p3kXdD6d#PXnCvFE&=z zN&kFV$(2kjtY%OY}adkmfQKZ=V7V&h*th}B3YoD$LjX9Htp!O5338%`-iEF6(@ zJ75UT(ZCTKd+Fd<4%ACxx^NJeV;O0Xc)zSwAJ8*y~4BG>IrJAe?=B5b@t_g=(;20N_D*R$$PmV-go64foUw^ z<$_Isqy}@6X)xs~7yW0^V1>J#G3tv?Hbt&bRE|pp3ePOE_SH^_TqrPIR(cz4{Z<)g zq9RkedX?~7PA&OEbaw95DMeBvkU6+hnj?;wDS19W;f2%R+GQa;&#g-HkeKGG6$<}j1QW?8_+RcdDyb@#%wdo-sF03e%o`;Z( zNL#y=Ped>Gh=?BU$s%&8^6F&V4VH*9=}m~=VmrFWjjc_BpvuiYVaBqCR=qNzBReb2 zt@0b8lkj_;CPl3v&m-t%Gsj9XI`BWr56oYj$meW%VRHwmb zM7_&mj`syz*wLP*4b0}w`0@HywNF^jw}`=QqTGu>XX=(A+JMge<{I36+)Ug-*Wm1# zB{Jzj7X!!PX$EE1w^sPZ-y{iO-cD+B=G6EIPQ?6ZUjl8i-HC z_2!K1hp@An!6TxnlaG25hLexV)VM)mZ8^Zvvvyk_!ebB@geS1GAUq|`f}poxfS^no zFOq6St~8#w7yX>}#SL`?{)-zg5#gGSq2S*4HAvC>PU<=$F$e~U?J)d4_jRH?1=7Dn zj!OQX5T?V;{z6GsyqUsAuk@Isl47)eMBv!QaST@Lh<{ zSMaR(U%TL9&|JZX66Y6u7|v+HgA;)Y&slKgH3iqhJUSm%M*t;l@q8F){PSV492sA| za-I+K8&vvym?%&<&W8m}0$;}YgW;s{3%MK4Xd#2se+ti8NaZz!)MG%-2z+filBL&z z%=w(%4de6$H=WOkPmw9cu!+vu6`q**2eDo@2eDq{AlAzq#4;mH-k(^ERds8QJH;Bs z)GAYj2=L=p9Nvm`B=#P`i~c==34vHu>P)fHHS6i5@g?DjBK08!Z~CX+gD8FR9aQdJ1}*&{Id#mhppQsgfW6@?50mdRb6Zu&` zPw;f&z#ce=rKrr8y=z9Kb55sN=BBmEN`Rf zuAiVE=_zsgbN?)S@z4DMacV#ExnJqhv`y47naRtEptQ0gWm`Xe-Id}|ggMaii}zWz z{oEghHLIW|LRA*zZbg;ZFW_~_xwZ_l3jC3B)e%2}Ksk0T_pUPKcuno@2frNp`Gd#G zksNzbSDUnZB0S?NwC}5j>F{!IelDhahfJ?EO!tAQaTUsRf0!Or?%iORnjYptDdctd z?T|kv-QVA6SRVju<0_Q(cvw#;_ij$Jo+j2eg{&vIx8nBC72SqYP>KdC&|QeZllA?qb1XP4mw{!cWeH&;9#B(%QxFWM&gE1QK%qGmQXH^3iD&Zdk# z_%DeBl=g1e-0C~l$o@?y|t%cUsg8;vbs^SdYT9d z`%V0oX19KGB$7AlmqsFSai$?%Q{#FXV@d#BFX#hZ>;(Ne?1+M{!0SGb z5k~k5st8YsrJz3Qp0zI!Ah6m%$GS+Ehj(>eB$%Q6sXjVE^b&8($X%J*OGo#d(K+JH z_C>v-x#IPdeS_v=h?#w>NKMWUYH~g`sj)J6`{g>WtuM=ZI7T?A>a~5e&bltH-7opj zwY#q(Xd7SGxnCySRJ^%Jtt~7+Fny8Q*M>{TVRjVFCV#0bij5mMuZV!IK55I|zPfUm zcz9A-Xw|O77RTv6=NZ_3u5^J|>$iJhK*j-Nb_LcAc#Z7NM7tluqh-QF6G zgx3ua09)jlfp9LDqcph+_HIz}S$@f9tK@H}JZ<>pBJ*Isk(VCHQEq?ZBmT2AZ7)oS zeJ-%v)evT!(|i1Da2k~utUz}y2Gw_k3Z@^7QqfGWaHWdFCH_dWj9=ZCY9J^7>b`zD zKr=V?pw;4BH74j@O>nvcm9F`F(DR6JjqVlNiKecH6dmb_<=)%S_s3GM_l!uqi9hjo z4BMrqZd6K7p-pw?vUs-jdC3zr<*9`E@S z^{q)1BWxBeG#lJuiu zgRSpK>X`pccvf$%z;ti@4}9c!j2uQ&o|L1+6-5nz7sOrwJ3nW)mqvzm%nshE+FY7k z`!RHcXc=T>>pPPtQ8(3HsrE7>G7c;!PN+X2+BEU#7^A=})K3en^!qtEN?dW{rI1gt z+SF&RAiMLKa?xih_j>l3X-QC1_J*qM@1KW3gC`CuvMqP_+RyPzC@&}$`$cRC;0waD zs!f4e0?>yyBY>CXC~-xxv3)4ukYm_+uk=}b>9ep8qWI&u0<&1WN{;cx;m)$rmx9}! z`QZUQurVU>HB1vC=GhH?h+l z&Nsvo_!faiRBy{s;wlpwZhq`clvOc&TW{va&IW()^U$vjh{wCwSv<61Zg^-@93IL< zb*PA(A?&5k_yH@Liv5B=yI1BHeC=R|d6=clRzu-gk}9*5@{99aG6Ey(OeWmT8;&u8%{{O9sfz5}*If_H6s11M zF~7@x=ShU6fM&u>^NubD+Yu+d_)K)MKbH*JD=_2OTlNF$`4}PYoX3evB{WG!fXt8H zX|7d5s1>w0yBlBtzQM*CF-rrZ2a$eSErlEdinS8|lNDqLfG(J1q2#QF^d zX3_YXtdV}hP%cQq&vu?&cw- zd$;f@ZO+QEZIntyX^W83UR(K;{?TSk9Ym>XNa>@_KBZ^6v=x&csCT=ClwRuQQ`(KS zZre)NQIxg}DfQaYr&QTyN_vLg-72KislunUuFa*>!}RXfA*D%OeM%R$x$ZU+rQd{< zJ`A+rm^M?|Sd@MnQaXnxRD4vo(OtGp=9`FC_mI|K1C2Vl%`z_+r79@Vb~@u$8>&!P z55gF5)DQN=Y5PG1y45yy7g7anWmDXy9aqXxVuBsD%ptTwcqg?>x!Ucdb|qK) zozyPmD%nYTE!PG+NiXFp-AQ^SSBIUX7jl*DqCt-GJ+TcUGb5@YmO%5)yA}fuv2@MK)Y$&0&ikd|#pZXZUiNXhA~n_kknLBaWNSpN zookpBEcJdWK!ejz?hOQ;{}G`{I}82s`|?ka$lVUVFaJR}t^5_}{tkmu_9vBYF$aXC zOyQT%>&+BR?=bgd8Rm(2#+Oj-dJcd@a$DjkpRCBozOL7{?Hz=@?Zm-^U^GKBM8!ZiCrv)%MQ~GyuM;`RXDZWxclJUqg3Lt8+NZ9 zL9DAJqNK;wWsks4`doI$Z4STcMRE4)l%cVM!Hv#zDc)Y-qz-!Wo4rVZc9qwNcTWNv z!t#Y6t7`Y~S&mc!a_@!hG!p0>g29CTqXF9A{?Y8Ae#9Q?KQw3r!l0cYkw9$5H_F*B zEB6=VHv1`GkXgIm9}O@su&O^ze7mN2Axj z8_NmQ``?Y5xT4Uwu@RDV!~c17;`K0i<3>pp&Q)IXAo2tS>g8ZPYR${R5_n@-gQtq2 zN(TQKTR09TTJ|B>Ic})7P~->Imt|k1Jr3fL6_9g$+;5(LJ@o&Q;R!hy9wvs_zqHmu z2^WUFb?{4T8^(WW?QqDtCrX+P@k?FVi^&#PWh^Em0^~-je*gS0aIu4p__=?@pyrMM zfT`&t<>=_DW*f`Q_Ph>_AEnSzS6u?1c+|az4c&Ax>8@9AhBMt#-|MQQHB%lvAa43~ z#v#+aum3nZ5t{YIHOT2wp%}XKHPAnH5`<~MtmWv?f`J1U?E1aa4;?ES zhLPNsOd1if3K$W$Q}*I+AW7mHkr<4rQP8QyA%!kV_328d<|zVicQfhu10HzGDwy|{ z@ZJ$Z=5dWIw!22z?rGV|im>h0#!0|QM)(eFcdzh9L2E(XJ-yYz*7z76Ob1K)I+b?Y zKoYzXA47j8=nk zj8hH`NHxj5tOR9^Z^wf$LGg!41_jHsA2Qve!mH1gsga|k62zqi+niJoLGYRhg^lL3 zl4-86CSQGV!r2!X&Lg>4A|a0|XsUaFx*V)JU`F(S2Iu}xqjEjkwp;@mlKYX(-Cudd zmeO@8D>pP<#dr#o*Asa;s%ppJVz?PtU%ah3D1IP*<2V3^pW$AVizQr*wv)|qYO|FR zrm`ojO_=|M_^6^48oq*f6HyAw`S>cU`p5ONFrtKxh;!hleVW;6q_m8;! z)y2U$ZHKQw_Ye%8@o*0{DKZ~ZIKqU+9#il{SF<_$M*NZZ;-iWdJ9j(F1$Ur;nN!)LcqLmb-RBr;O2 z3{FL?BxP7N5!hglZRH*f4)kXj|8?%4!EgGl==C z?LU~GA^d~WXDjCvaw1v#V!*$`pENv>Ea>zXWfMg?c4)#AGt^kEq_~}tT8YeG9u7`Y zd{Gu=r4f8lrXvntl$l>hjIq-%D4L3jKS01c9joozW&Q=fsoQX*Vw)eUcelq{y!9 z(+RbeEeh@#7`)w~{qnPd7*pB0`Q!&?R`zIBc0H9XKzuU_ny@NB=cjoYVUH?1ZMaOk zuzvltvh?%%N{aLP(jA+fn0=kUMS;<98q@;MCNN(M%!<#^0vj7VbvJ4XwddeKu>0}9 zziF5;0?jncAh&Hz;X;_-nNL}AV<2~+)33rg*TVh0w>pfk`F-03b)WaHOns@L*%^={ z-oX9ZjND494&$KK>7jt<)A2|D|I#=(M)(Ujvr)v93 z4d+}NJkf?wsJ)yDnr?aj72YSgGvvwWY0`?XVPzI6;-XObXA)}5FADDE7#x4qsh^WM z@go-Y#Lp?rHD)V?U*c2Ydb+c0V`mP5H1WF~wV4MUwVAsezCcsnQO;92;!T^WRWyU_ z-HzJK87=j%Vjy7Mtuee2Asdu=+^bzv$6y4TxOIxtG&sp6~e zqhgcu`{)fspw1cJM{gt)9Wn<&W$SpSfnt`eS6d$DsHe8zFC!LtE0azU{XJbujc4Jy zZ-v$h+S(auR>Iv(B;h#c_LWz?Ftdkr(wEGlr1$=NCfF}3rVB}X#uvn zZu@N_c{5)37K|{$KiR7YPl-tz5bs%gD*=+iEQn&w#&T{`?%m5-2k7(d&lSFWw2%#j zja)gfV%8M`H-~%;o-!(v{eaxPcA-yvu3|}b1$LIGb(+!~uhlt8X4uMP%6YYN<*ne` zNt#{3clccADL-7V!Or43U(B@QbFCbut}?OXb7*;32zr%EgEexwhBR?Qc438Fr<^58 z$=VIXcq3sD)j3K1)Xkm5(ML6VqdzO&^ik0p{SdnuV&=$QMA7pexywF=AwCEy=f(cJ zh?AIf5>vJgU#j<5M7R-Kbat=6beG~c7JEvQtmbfLsVj=ea%N}eBz-0SrB_kv3{6## z=p0QFwCga30JWgtV0Z4{Ba7kYoQhOUr#zjz1uj&2Ld_A)yvm|1*~)Scp(^zvzw~$e zrC%%tNP)hdU;FZpl_@Z*^!Jd%XwqUiN?lRp%?nTbKxk0*s=XJV_}uIBxewd^y1HCi z+AJYOzNL*ve8$oy!lF3$qDGlA7d0_P%`9qC_X{Tro3O!CdGS??&pa5)VpQ|2TKuPZ z9`@z&fG>{+v88p75T2I1zsyizmfSsZn4Mf5ePu!+ij8H?{2W?0FVMPqYR7E@9*)7% zy(O>;_lk8!mHJlAqV4sYI7@L+gEv)cMsfuY8OvRnTHtq};Z7iGs5cJw&d6~aUvEs? zxQz^&quf-jIf^+0+u%G2@n~hN)nK6s-a!l&hBeB4Z|~Upe;IkCbdP~l_T$(R@>7Io z30Z+zLViMJR5xi()0VoTNMWzcA>=CqAzz7*13)XPO`BZ?8Ep&cy#8fR=Bj8sp2eluo!dIRD(*69Rv}vLx6C zSi(L)di`bSX_c!Ei?y!bW}YXbs_|t}!hQu?)%gP9>FV_RDg~yi^I3f4SV;~uw!SV$ zi7Sfi$GnK;yqT6&FtvG8d&}{uO>h8Wl`!Z4M5HwF0}vt#{EPz-FB2;4ZzwQYUtR|iCM^E)iu4@R-lB)u< zXfuIrJ=`*t?yWgp!3N%88ee-LCNM&op-w74+50oGK zLu?^`lkhC$3d}^XJ}RY4++mIz5=s~|37k=S&2Dc+tC$8ZR$YG9JDQURdDL{A6U-SIqLN)qiQRd zI9#QQrE|Ewr>fK(!*xC;Ti6Yty6*+kfJ4*^iKtGDP_c}>(LI|_h>^J4^MzWr4(qC9 zpAw!`G6iOpOowt!IZEXybwyFF3}QObDq^4c#A0Wv6&NRGK9%e2YUEV48K4Rt_H{T@ zdx;g^mvHrmeeMg6NPLdpANFij z|G^=7>PV(+9mfn~hbc+&E0PdXNsJk{wveO5RZhm0LL7n(u|* zk~FW#qTIBeYk^yEttk9OSurplB>kV-^#!*j03w=VLJQUu9r{F58Y{dX;Og&MdD4{c z@h85+@YO{}J>8%1`&(E4!)aQ~UTa7y2W@&RZS{eZBH_){ehZN9PxFhW?6d#Go1%&!c^4V12w@t zL7<({L;9)vL-8$xY!!>C{OYcJzzahBXB^J@cKa8 zl)Q`tVJgGgSM&#J6lH`ZMSq|s`6C4-t4r31qMOiy6@5F1YJa}MD~GGE=#NOzH^!gX zNJb|a8_MX2;p_b%GP)J`eZB93(`vK=-AysTygjLOxwH~5`rPbO09o-y*cnd&>>y(K zv^o2xy67VO;Of7p0>rghXKtO%2&Pt+)NFwxTbmCEw0ViN*`}g`mW0~8$5!!hwE3Ha zwE68sW$Jyj*^@Lis{jo9{1R*0C;3WyiJRTalb}Oirb4$+!YqYeVia2EiKWnyD$c9W z59Ls3iAY5TwK~)d^Zw!K`In_NsX`2OKKEl|n zjxnRq<14)1z{OYSN2SnP;ZJOd;cKu?HM`yM`x;z@(`v8+-5wZ#tRoecIdJTW1QVIs zYHSVXBg70_M~|iEy$H{z=2`K%)Lh4x+oWfFnrHu7pE(fJvfjoZXX!0JOOIG}Cfk0~!lU!o?5Fnux*N!U z6k(FTIg=fhu5YVKr&PI07lt=qIxjBY_!wfLj}}^eiW6QlqcumlX@n8A%-0BW40tMF z*fT`N6decmf=F7}>|V*=k`%eN?uWNYZ&frdV&j@YK64j&6U z;ssm8!>PkHrX!xp!!w)cW|**sO~Vst8g%;Jp$d?WR!P!66)E&(%p4vw9j!TsO&v!B zl(&wT)?_^G;zNtp`jvuQ@4k9UX)0t`N*!T*r;}dNi^PAqgLEW!o ze$#tIq7HvzR~f@(43)8qjGbldBx6SmzsmojD&HNyU*&t?v{kM^_YW9AKZ#U!zPl#@ z`RBV?DRRwsCyHRc8AA4iP;?P?xWZ8((v_@U=oEvJU{q^a&0aXN74x(}F_%d(Z7LLV zS*V!Lb&H3im=BtT+)^I4AIA2sCjslX&jGW%#*?H&7@kC)_DS|>pDH2kQ*DHOgr{oZ zppIxx%6gZ#1zBYE$@VwMPn9%K@rZupQ}LX?^E=YO3VxUzaQ@`@G-w+!)sw&rQZbiU z7R)7<)hNnHR&$AEHOUe9sif1%8ktL&(1KgTB#6@EO|0<7z{PJ3)+hdgzj69fHUF8#NusHNqh{%^WvJRJRB$KI5F__#8) zq4ZxOW#uSBo+KMC?L3J%gyc!q={N~J^tC+eh?O==!e>hUBsOf6gNPROzv2L-}t=76toX#dKO7&t* z3P-j|hV#l7rII!kD*0lll4o~~hoh4G@;TktB)2j(S@TNOLx{#xg`|C|nNVFbJE;~5 zn$=0YXf%4EN<}9XQHOjQo%0oS2}OlOs64-q=?}xVL)#ShJQ8?8DmtkbgHGzj8buii zsgruKCV2{eM1KcaBb}59Em-*t@Vht9<|m|YdvPI$FZF0Yi8oeR;qAJ zTq&ipsSw%9P^nIv9?5TFP^{ue6%O5sNIRcywRM~yz!M1goq8sfpRn&CdtIJ#o zQLI<3^_sO_x7I2wX8~DI-&HlY;K-xotEA*M6{@@{RPwc2kw?N&@rTTc{5@9WhpFdO zorJxkr^*Q-da9+gPt_HsY`Gtel|HK(=xq%tDuRZ(qU z4Q*5S+eqLAspv;n1^wu%8buj-sUKZcle`8$^|6Mmk$%*K7The}gDB17@(OPeT>NJ7 zrkcf__!D=?xLw9V48KLJBaeFze!oR5#%WuG0^NHtfcpWdtghcjKz?1%N|8&~KNP{X z>AL7fbzOxBRmUu3{sB?I{?l6jvev)F>3-5uQ)_D;#F4G!;r{{8;V( zN5WC^4@^J)DXW_kw7QXmLo}X5o%TuQ;hMc=wT=}uORd)$wfr9gL?P4%5pJ(W(i zJWQoP_i3BzzodfX4_MAKO2M~Ddk@zWF3q-(nSD0%$7rG=F6(j&uD;6Nf;)nf8@%iu zWAbMCHXwTAeB`((hV(AV7vsqDAE`3 z`$#*H$bALBuR|+wS{+iL`zi(%zKKXV$#PFQg_{cdT1Vpet(nTy*E$myp1f(_z4kRy z=@Uk|vSD||PQGkORp2&fLN=A7#8q0(guIR=zQK*P)aJ_DN|s#ETII9cLfK&1728=& z;8A7#@<#wW_J-6C)tr6#m>(1S!Jg@>`){>o58ae=Ejb;2#8ecNi1KFqbSZj%sp*!s zRQflSo}VIZMf$CTF!>f4+_#lWtkCB!CESdMqN^BBf%1AHEJx4UcW|j>TazE_H)8uP zVUkghobKsjVa=!;`?%>RWs-QRd%!#r-a&&Y>oOAANEv+jU*~Q@ZPT%)|_EicPZiAgv zZQ$-Fo_ZLkzZ@N1rG1eyxe&iz=M!ZV2D|SG+&JZMUH#y2&rR9mLmT>Y()!_6W zIp0H>OI$T$-Wm9Z_%)a(2mXoEc1Q|z|AkTC&Oh=#l1#1bBidEf{##T5I!f8FM`Jr5 z6X;CMaVT-RIh44F?YZeXlxT;X^Xgu%9(aA1m#WRzHUExcs$K+-s^o`vA{|ZoeTfVG z$Lcz2n@{|rU+2;tC%xDwKEl0yg!>3#OOlM#_`akM>ZJjKDdOTv!Yh#x|Vh$j6`b2EML zc;)<>oDO}kD74TAPf+@Al%77g73u#dgvoEo;C`oEVioO!72_$;^uZ!*`{3_!sbmL} zAL|gY{V!qagM;KP^}&aWH<^#HG>*jbL?g2g{*kck2>RgM5%j@7ki%81$RFjiIT|}} z9~{9Q?SubBKBGQ=mZO6!;;2tsOM9GPWEX=z_$MFe@k#;U6R?x24csS+r+#4IuX1#7 zl@{h%eelW3n@ycQ_0tSu%)g}YQ#urCw$i*#oAGz{i!Q@AP8j=8>WSI!J3|C{PCc+khN0Py#Hr<2)8 zWriFj$#xKT0}eBN4ws8VOWo`S>5g%hA&QJe&D7XzLQF+4dp8r^f>(f730njE1tStA za7nbs@aODT6XkZm@AtxGIBhSiK(`|XV!MV^Sl3}Wod|N8I(Z_ZH9CTu#k+Oq_v}%b z+Kk5t(}PfPFXysYx~t9mc2{u?WNQ)*B1`dp7^C8ESth%09Snxo3%DnaRO{f4uvLjS zC=~l9!P!`FF3BOs<=~P|9R0?!A zwW)3w0^VMgRGaEPk}NV_Tot@&~S$N$E~AOBmVJeAOs^dlVO z(%%k&YD&3SsprQM-ir9m;?_m_d5M_$TFaH(t{=>bOcjdd{bISH0A3j5JJ(eE7d(p9 zRr=wn3}g%@h5nPQUzCZ)1JE#S>_It7lADp%4JzIvOclRGj9MsLv+5bbeNno87(^>wyCT#9idlCkpJneNxUeEUPeyvPo{*P>DVJGdE z2CPIv`>y_-TI7(ieM-n8b!fvTR;C8hpJm9`KcZGb zUWq?drk?meL;kid@`u~=1AsL=fVB9PAre$C{`!zx$~%xxq^t=fg0%#2`H0_BP$>Sj z!NYV}Y=j~~)8)2yJ#Z3hWhAMg-ctQtUr#~)ii($3#&GCT(P(F0IXzfMP;;N?oYozA zm#qk<4V4M4*zuP#i3K;BCWW^b1kF`nG9Q6A*)jQ>O`18tN^XNd;(j4!moJQsgrM3D zPF^&4+!>>}g1xWE7>yQPHouF(bLR(F+VmablJ>icXcsfqqxg%{<}+) zmQ~um3eAi(6zs~KpGlEf;yYCrYhK>BYbjJ0RK)J8h+TP(E8=BU#4fyBMQoo=h3%*9 z;)3V#W3yS)^ZJ~4Zlvr^(hEvM%3tFxIK*-^RPIsl=)uEUa@OeFQ2A<-W+_A4d=lVT zL5ja+)3p(>XiR(^XfcbBZ|8nnwfC-`d}*g_xq7k+?Uf>B0o7F`9YftwTCHt&Ckh4n zswpLoWvT9t(wc~42i55wCXU~z>HRhq2Pd+Jdl4OvFZbrrjV~1$ev_Y$(%Ked=i2RQ z5hJN6ds_Apg)<5*P`0Be`wnoIv>RnFNEg4wyBk1^4)^6bChxmEI`R}5?#E9@X-t*8 zeJo-uixPa%Fyn}gkfQHvWWE{ z&z_c@yu%ET&eA`cr|1%XmbAa2vHLszRDU05TFQC3=@WQ9)U!xB&Sr*E7L4Bd9}!_nM!QiF14eUvpvD;A=3 z#d|>6v1*3PZ9#dbE`I6pG)GY0-olZN1;s(>j)bf^;87)p&&?Fr4duUub!dcWJ4l(t zJs^hpx)ha;*H#6CV}@k1e3LBqir%YP??MGxz+Y8EdZ~0u+q8)jEG#MtttYjpM$QI+ z=~J^d0CrIcjd8>9cYumVO)vB4Moo$gf6ou2rr?^Qss`l6zLGDdk^?m`))G#gptXca z;U9R7!M?(y1FOjJRep3=OYgv{$tGYo=#12&H76~<#sjnT$^1|0B`lRqU`^FR({vV@ z1gLAEnSyvW^3J5aYFgH??yLeuUl=45-f< zk2%>kxNv9&aMEjD=*{niGQS6{n#q-PbPxHnJs*-Euyx=--L5nEK!kAJq?eh9PD<5W zJV_ht`vT-#K@jYSrl2NjpnBvlaXw;Dkm(=iZGo zh9=Zyb~xT2>rRG#UsagsFCLX2JKF66m+-%O$zI$Me7*`1O5c+%u=CGFG)@_Go5#tv z_vWQSuKZ?OjrV}NPOE{>rO}H;)yt(;G=gQ!ecy|V8U+!}0DD4*vA|}`&lk-M(qB`k&5}l~d-ov>} z=ZE*wPq#E7@zd>Xkpa~rBfTrr{=wj0kcPu7q{ZG(2LvlnBoa%|AE`Xy$e!3^@C1yJ zF_42gBSE2HxrvZww}AhHgAFn=LYnOF{*m2;&*)<;$rbN^CTIN2o28VtM&de3hiDjg z7!@vHIySloS+^K5!K2YTqqo0$ zS2@+RZiMInnd+Rs3Cp&VH z@X#u`NZ1BqAk(EU!52Ok3@4RQ!@@}#Wu9i|N-pGF$yGRsZ(guvLAuRlhAMd%71LbC zZh#5qy3Bw%6EKU~PY28h@+AbwzH69r8CJdh&n>vX4X&)*($H|iB*D-{F~OCY%a2du z=+hG>)n0xqiUDxpB&`ZuY5rBZ)qE_|t6+q)%y(GS)ZPOpCjP+^;ItS zNG|yslC96Nem1^h?>R29%Js96cu799l$d+bHPImjjd=2rHr$;L~qqjRnMP2#&R9b$8 zpzhKVtRY2R(tdWUn%%j!gL^gyV4h>mZr6fz!+UPr?9OhrX15q%*wMY^I$GU%cku8k_MNpZ=9ku@W>B65mmsCl*OMdQ$TuQ{t?UAhEyZ3uCrHjxyL+@%H}KAyE+2#@?dX{c05a z5-GSE=p|Bcl|h5~Kv>v$dagAPbQdQAc5tgJ;O3 z5qFuW?nd0XbhBtRBkpGfY;FeM)qZp`K31%FFQ>YwqKgai!xZ5|wv4Cwy}}Jso)<&G zHW~QWc*UI{lO)EHd5qQS=jHjOfliM|7z?HKb_SrNufGve!ru~_-nNmDZhyLv$@)9u z(^dO<-YNG2<*fbb>LJ>n;Hv#?oMGc4&&J3c8yTuLrnIPeIi~9Oo~lhqD56U71yt!O zBBSaxN0s0*gXK=cFYtsS)Vs<=b_55Fh5* zs;YA+NoB!Rfmzz|d}DN=^3WS4m($2_`~@1VKHk(rV<<+XC9^VQU+%S}rH1U{>lHI( zqu%A2v1D{tk|gvNq3P}{5=?srW`}f&FVLR9d1~!aqsCBhp%#csNg;TTKua$v!;@R) za%)ND%_-a4svcZjxN_xAtMjGgIle+xm#{T(t3Xl97%L?@-{)1~KL||%?Op<{%S@m;fYvY79b63+!|derbm-s4fjDv}JNhOLR} zZq$%VFB&!IRws=bQuuPChS=#OYpC<12H``UGNT5&4{93{G^|aI+!-M@Wuj78b0(_A ztqylkxTmaq2G5#Y*|M5zn53mPPWOOom>XOlENf(Wq+Y9u1j04Nzb7y`a%C-IO-)AX zp)nL>Y;2^AFjC%@P^H(2p?p?Q=)bokzPtWAmtNF=e@;My?K0B2&UZdxjp`TonZlFm zGb1k#XZR8e$F|j%V@~YOCOB#VxiyDQ60AAYEHo%S-5loePPx&Pvrj0;>0yOXa5+e! zi$II3zM^GYx>;OJt$+IMxypC|_G-1Ws()1bJ;ke?`PHiYfTJor;PkXw6^TRbB$Brm zt3Iro*&JLq8THr#Vzu5L3{tjGWeC@mK(<;n_zLkTTh%K-&&`!x^M!I)-52lT@#4bc=78m5o#7+{#OEM6PR|PW=&-M__CrWTy1!qS< zvor2_`H@}!TeBmMrrc~3!1YyPXO-B#`w~d);wcS$Lwzy+4yDQlln&~BR#AfszBF}&V;C* zMB7r{J%;}I75FD8WLRHS44}|?3uDQ zSH>XYm{0F{kZk>IEF>$LkU~{o*NTW}6C&-t937mpsdB6ks_AtNPHxeeso2I))?f33 zu8fJk!JFvyYD{$T^tY=0G@Vmgx-rq;0G%p$7T6VTPG}~24-(QQ+E0cRUtpr!MZMO7 zXFE0oHy-=!=^@2q9}h|K1(3D?KjL^c_W2S(8T+u0yEj9eX(vAx)9-bujOK>RN7X&7}k5Oe9)FlsgscLtb@%vT8sc5%V31ZoVnftZ2w zzu7>n%9Q%(X-Jmtn~FEHRTImg=zarnKZ;96^J7_kjWFC_k;}T9;k7N%*un$& z03mJ%LnMU<@&m2AL8EmV2PrwVgW;#Sicg#RZ@O|{r<`>fd+^{k5XHW2ASy~beTNc( z;?>EJ`Eo&@&D9>C_(i@@$BVBZ5$W&?3=#U51~`%T5NftXS7>;|IB z(5%@&6n382KooQrCq^yK8vS!D3{u`GjrANG3XBgisG!Xc616@ac0w} z7^hZU3V|Ba0GEYqX@FMeiL+&BWU8<(b-0jeXL>*{%7(YuTWk>rr=!OmtCzqbk`$h)#_5w z5D7jD&eOD!JgR&iZ?a^UDUI+7LR)h*SLbs`C2}#5)@ZKMLu05yn#jGD`f6lJ`+eiH zTiKX&&Zo!{gj)QJj|9NoT=|*45s1k+KPKbqn2c+S$k5$NM#c?}3>A`rj5%5!KHj*5 zM`sch8D7c{wRAVR`mzb(Npz~BO$a0yi3g4Gay2!kd>Kj7k$Z%QFe1$9%DqHdCEcfo z#!!V+Nm6qy0bd>ixjzQ-lOiCx_sD=e?0~3{fOwLtHc?-vq}OGyh`~P+gMYLL{Lel3 z#~gSSl0IKW7;t7!8cpKAUvqj79q2Qy`kv&OR(0Nw1F7w5%7;t%A>vm=7(i&XBb3M& zDnleJDyAcpdXK54d=2F!_Fn^;;0YnPmN+ynfyp-)Kh*j1^)%^83aQ;jDp**}af7@* zM)s5nfbAK=TwOm)q#gD5ydDOJD%3To|2kNgxH07$V!D14)Ad3TT`zjNUUGD)keza- zx(OvmIQo5DvLUf_@fl}-AR#o%*Ke!;CqU5+;kI-ONaM8`+?KffhWFN+^m8LWvHgFY z?7@^_!EHn_F?f2M)>xk91bIC-SH3KR8B8rmw-J1&l6O&7b7}pH*ti`?vC+7LM`tt? z8Q#ecQg}lsN45la@e0aM%z6Dd^P>F(uk(t0@Pc&;(hcQtN(SX8&Urn3yUpDIjKn1k zG#FC@-~{hJdjbF=i6I`gMv{gWcj>uU@()HWW5M>#`2ZEnYMO7U5+c!4V=L|LS?edTs)<(0o^Hzs$i6Fo&_1!cgSFrX*%ApRBvtSqQ;vo5 z&*XAAKCV~oQ&EqPooAn#OV}TPA_KNy==x9cgg=3ZP6WN(8d}?LsC2RVJFomxiw(Mn zTw&nM$^at0r*!P<)}ya_t`2I}J(y@LZ3;F@TG0#E%3h?EE!tvcb7dzRl`Q(wOxz9d z!a8kcV?(2zu1C3pgNL9mUs0l3_JvO@AK=l|nHUTo=ICd65MI7{?PqPnw0x%47SGyq`X zV?c^6(c?Ti87VUS6+g(xuH?ZI{hHVAEKx4K$Px__{Z9ZX7+(W1YkI`DH$7s%YI?l& z7!0J&NT8^{^ho1LqGUBzPzJ&#aru-Ym(_asN?AVE<7r57QWjC+GyFj1icl#{eAcI? zt%oA3n)nM>Lbnu|^%$y$#!!4a>*49`!Ft3M)*I_F3>1lORtI@vBboS)AW!%lMC7f< zZ!NZ8*2D9Z)ImM#5o`W(upXj1wjR&(=;}-ihQHy5I%|ew)!7SiYQ}mfGFNA-QzB(O ztS0+yJ#>!}x{VL$@4y*b4}3u9w7$rDY&b@eNkf?56V;vJ$fXw<4!c6>4?uFBi9!>5 zr{p~omo~+QnIXA5srqvwLlGv#yVv>-$mAFjc?m!CEL5zDBXwTK$nnS0{=L z|HKcu#t1mp>UCbbvsStEB5S4Vn1E@wG5r~U^4m$txQ>gDMpYT`S<@!Y#Z%mQalyGH zSv7F7jQuE{9$}4#{?160*VxFv5GC6fCKBj2&ue%VORQ&I=@Y-HTXmxGNI)v=BI8;`Ee#9;U~Kh)VYI98p#8>eP$ zq#|>5Hk}eF8)-G!ZyPxSpy>9vwdb_H2h!H1{KS?~mxD!$gvx1Hfri<0<@)vkF&L1# zuIssUGazZ#_0Hgr>qJ7t8Q41g>az7>cbUs-94d-^-h_Zby!R8&c% zpFFxl!Og>LzPUQY@qBoD=4f5I$#qRAf@mYm(?jtk?YuxyE%v%v&t z_}tE5-=S_Uvl?>d9($YT*LiZ7%^WsR0i^R&nZZ!qb| zzpf8nF5R>~?St1DJj^0~B1-iBCGWAiGajpk0r}!wmmajQ0N$9VCLYr~4b&$w$XD@& zJgKblv{{~~&fpIj`CjGadzIw75+$MKgRFdq5F697GLJ>HD87&ul{H#s=4p{tS(Wte ztV%9@d0Q1BfaOA~GL*EK@nJls8Moa#DnapujH|3MzB!DC!wr&6yhjKU&8%UDBE!}A zpUo^y+8zG(#PtOc~uM<;w1O!*tL_qS?<^+$DMw#$Vy9!+k{UTy7cJj(D# zjE1H%&6SmCsgYW$Uk^|f${@guj#{cxR{Y^oM&ofzl-4E1BWY6d)-y1PIcYw81vTxs z_TvXMXv{sE=tn0cr!ZBvLw1Ry<(GNk`h!uF4t))F5~mvMA`Dtc>CItO2+@x2s%)=P zy{wvG!SF~3;zOk#vok;*=?HGYv0&C={B5sa4LZatG8 zN5W7|qC5|Civ6PYhc;-gJgj5vP4m>!ccMh35~h4;&Tf1Ic1u&FeijqumzH)GU{+z% zIk)A~n4(o!b%;=!UTXa+#Z8lJW%Q(;J|*^89KVqUS#0_FkAQWdr*(g0L6p|dI?XU4 zIwijO-g*m0t@TDM^TCmFJ2qPz5!51+@sfy0A9tbo)=oP`|35Hi07zzZCr+$upTu^FlzQPYQCAe#9~Q14uD10HxaVq zoi=cWE;yER{%{R?Ju#A}^YK7DZ%s(#&RflhcinpADZ$9w+sOOBjIbgvtVmWnjdeG+ zF@E0)o~OoU#IRmHJW4P;-!wdLX;hg~lh^#H^6QK$Gnlkt@~g36qEV)+e^Nb6N-#{{ zGEDDeIJ(_)bbBpFI%D^eDd4#&Td}S6@F>CXeB1DJqxKno(jMDse%7ytM+t`KJK&MJ zyR#>y?zDEhmUcjBZ~dNEA0SRTakZP%IvMb-8}Jji?E{q*DMEt|#$1d)TFTRSkCc_( z+*}yPS(G}$gH(WU0b$UwYOo=>lDSC!Q04e5zReGI!z+Culqpejg0dzxw*Pvtz<;|Y z^L_qS*88-zcI=RgbcpdrB7l3z?6}cu_!5Hw3A=AY^tH?9U0QWawXn7fq!%Al+J4j6 z)uc6J=gluoo2`E7kY1Bi?uMipbx5j~=>ti+_BS|^d9zMv8HEXP!d0Vk{hhBwjqQ?J zhOPL6u)`>y41W&ig`Qh=&%Hxa6$}U!^uL?YHNhw#U*vBJyttd;i~KpQoABPcv3}b4 ziIuV$ zKxFfSRo{zLaB~&hI4*dpKIZhvCnm|8#mSdz=FCN?o=2H+XBISXLNmExG`WyAKGRLt zbQb16&%P;2F6>0Z{pkUbXo^Wss5v>IW{a4bD`attvg!im;JktZXfN<5&R?L;NHwEo zg;4&|k_Hq>m>$0bbt(YEb|UlZrE@H{cD*!#ja-UWKYiucxT2x4pP7*sQ9+f4ZfgbMoKD9!^aje_$Y zz688@M~L$t=Cp3bd)ySxSEk;6*`7L2Bi#kf1Ds6u1%gF*Az^iVHCN1(Di;xHQ{7AS z&={%^Q(ao&);x9B3Uld2t?*Le4i=AcOMC~=0C=A;A(CTtg5H|Dvqzu*%hH>>cC>%9T^h_5NDY!|eou zY8|qp&fywX`H}K%>+;<+#Y#Zn4O3L^hAH#PU6$XDz}2HJn8X+C%RD-56uhvLALzJF z1Z>Z<0aWW~V8XMwY4W1Y$4rt(Y7(#8GgI-(+Op!xe!8MNqC128Uw)LQnoVSsb)04BXA{2-7c;0X31zLMb_+LEoUWgek!|wD5Vbb=9-*$Ce{g8goIz^ z2g-f~W!eMT)u-CNl@{vb$m$+Q++=Q{gnfN}haOf4#cS8#s`z?5$RaDeWp`W7!f}=0 z>-sJUd_I)rjq}$nHfx-FXfO?&ABY8xn|z-v@a|yC*M(?^b^dESx@r-*;com;Eq6nI zuxVy?^=6&fRaOkN_YeJe@k7Ro)XOVl`M8~5YpwSHSXKLt7_Hm$itAgEx%!S&&H9w} z$9iZCWi4=8+Jhi3+`TRKe+f-NgtSrO3?cW-<0=!7eB!@c-{k^7dbaJhce3-|t5 zxZm<<(~U`yIpO|<5~;6yt%>z};`1;-F*F?LvJW`pE|vpb=CppB_qdCFgiPUgd5>Kd z_T$lsN|E9I{2;1F$)(=#0A9Q64Rh&5z2Pqf>~m`d2RdMG`|<(S!`yoCWY&WtQ7Cij znzL4eGr8-n20tdym*u`-&mTA;XuxKVftGR|qfH|;)U@gGz7~Jj2CGXs>Z+accI&!* zY45!c<3oP+&3+Gr)C>n}E*pOw<~oBr-m&TZS*y+6gfS*&OE&=`te26brO;XI+nz0|x+?s}T{95@ON&d%3}pe34jrKwJG zaP|?R%T^)C!7O#Mrudp<{1)gv3?RGJvJBE1ElVNv8rxfcS#MW*F&ND(a~%nv(zqAI z7U4@0-lef(KhKEF6S6#*vi|OOV3X4- z{-eZE&QMe=_J3t4js;sveHr1Z;};Ea$A2`B&QOTl@ECp&{$C}U-i@kq8lE3j>2j=i zRP~05j;XRN5L0C{ME&l^-v%mz=7fY302{9#aYDkJ*5i1O`|jv)H)a$q`ABd87)lP>d<eNv6_&xJ{Agn*I40s{+UU6LITD(HWMY5^h*ietSm0bbmoIM#bk>qWe`UdT_}phim=m+`I-!>Li9zlR#7 zN0v)k51;dn6L7{99w>E`Idv%#AyvDoS;zzx;nbrmNMVdo_tc|Yx{WbvPCc3^F#t3q zBv{u4>k)`OJM}07r~juOG5+;D)lBnJ4izzPFC&eOYB@C}rWX-a05jlmf z6}}P^?e0Yb)C9fgHk7L%#91zp6kg2_w5|`0S}RM)^;eY68@iqdG^y?}M!9s;r1UXHf^tI-P;%)xC@?rpB;5$4%&A4w6?;={?1qZv z)S`@nW=|~=Bu1BG;DqyVbWx`he;N0l{FQWZ&jDKfh$%Pc ziF#nR0*s{fV}MA&ZW|#Mp(1l4{30cG9hjWqZMLG{GvsXmiqdg@^8?_FyHw6^p3{0i z@2&Uo6L+Z{;?Ze_BEyIILH^s3D|m!w^|~Hp54~>vaa5)+$kaL%^}+U zZK!M`tkQf_+h2(P>ViNdF7%34ves!9YEd`<_$^8MnFY_oODLg=&_mEcp*s zee?E5AUzJeP}Hd!z7})2o4EY7NPck(2JK^pRqljQw+_qo33fHs*U1w;p@LbH9jP37 z%%b*s6ynvVucMIf4@s&Hc@@rrpN5{4?!+s7PZH}|uXqTb;s*ij4pTBp&-m2zP(_hd zqx7IF@eN9tdhMZy#!&J^z1HuxO-iG_L;lY2IkE?>HJP9BPMOb8?bSxmEw5<~Q;Ju9 z-jTVFuz_`7!r;&?oh!yYR4U-dsEv&=+x}f+qD@qVDrqCG{ZyuBhW00$WcM4&RSkN) zFRaL%><*wrYU|P@+HZq?2tZLuPJ4a{oUuXYwC6dkFYw;_TYh4LeyI3*nRl6LpScH9 zO+)t+BqW}m@*W{*%y51W!0u)^xpXttH8Y$e#RS#$Jwk$`TyQjj**aYenSqm?;Vc;D zyoHX$9c^B3u8DN+t1_ErP~EFN$`MF%(`=3gRygyuMjlC!b$gSda?YHLD*u6k;`NxA z=L%u?iXyXy#of1^wxIQvZhc+$)tD4bXXZdMq?b*Z=5K({@6URbBs` zD{&kp%uX!i!A)tzxHT|EX)^K`BD$N>aiYlL%m)1caN++E6_2yt;?dQBBE!G& z1A(Uq>DxSON|V7{Jf%66{C3i6Mu;`T-|{;ElrEm8+=QnKOJI6EfcoE!-fT#AHDebz8kOTxJT3zrJR|F~^= zChN|#-h(Jt%OWp)pC3eafrJp)B{4GGg_p#nd%M4rN%eMNEWC>dX>7#kDVR7i_%}I{ zHuff$0J!T@ccQ1YDb+`$*UZCWc}H>853nA&ctWPqZbF7WFKx05Su+@QVQW!)yBPA! zv{k2VeXC|auG0)JgE6lWnc3VWdBlBF@!leiVt(LqM5%EAnwg>FQq_;mDpM7momJM> zgSEj8`o`{MpfSgNT^n%)FRY47X}Si>od1U>Q^NFgUXeL-zF0-IfPR%88bh^OZf5|- z$1#k|d{&V;7*|s?HJ`PPqB{Ip>+*(2oyf?~IHFUZ|nZ+d2>T^~nrO1+Q!IRG!7rfeM47AP%#%rGYhH(cA+C93C(x`9{pyIB5FptjYD>7VxpN>*Xa7~e55y8c5 zM4R!vn0@Grv`NuK399te3C?($KbzwBD!4Dh%NfgjF4qYvTMZ0-^#4v>Of8>duNV3yibKlX&w;A!m(M@B0F zI&SU%(L`rO-mQ7>Q%$2@-dOJ2B9iY{rxsfu{gw+5_L&_`j790|oU>7U@IEvlU09rXpH(x^P9Opy&4WTGK>8OGi$n_M)1ha-IhD zn7my&BME_*8RH0P5WDg}KOC=@-b{;^r|NArCMAz}X*z%qr+6YMoWM^<>1%fF$V4SA z;@ac*nom0pjaS8|Z{1kO%Iv0g{SCL+Bv!T9^bIW$P2vTes=IfdOD~wR?qT!lJp^uY z58!g?1>p7!rT`_q+O$_NwI*h7tukd4H@nKD{3r#hOv1<$t4xCG;>482X@K{wa2l`i zFmyVPPT-0RH{_?I^qpWMLai=kFcz*Z?b8`4GnPOq$~gd^=MAOWm;e1T-3+MzyJ3AQcI+l4tP3rDC=F+WRv%8K5 zcWQehlQXxyVbjM#V^B1^ON?n>%)HuvdZB0rtWi0K@PBC7rkDS9%m0Feo9T4qfW9%Y zP8CH|xCuWUrNgZ4Z>pqJ+xJ%vDL&u!4_9>_L3N5#1_B+WBegWtosyMjk+oNwl&Vge z(ur6E7O4JXy1YlbWeh77i`A zI(4d?;+fmpGgou7m*NYU`yu60$7|WhqrVe{%9#K~Ef$Q7wgu~w_P=W-^TT-ZPfeUf zdXl6**L|!o%Wn4Cjw0kan>@^1?5*DfUEm1XZ)DU}CFmcTy?mK>H+!jEs7yNVOL4t9 z@2lI&@`$MPzZaeNwF5}f=Y4I@yPfx?1lw1<$mrp`uM|Dqp7+&_3g)P(Iq&N$^{7#T zQFC!m(#e0AQ!p3sbg0rbUH)hi-pUV*M@SEhFPIBB9Z&cCJJusq2}bB8JrSDdfQZmM z=Q7rx)t1tDcB+R*35Ms=KHD87&ul{H$f&C{X}{=P|ica@M!U*44<1obN6TcpK|f1AfNc!Lj@Qp|{HQYzrlr1{R^9D2;OPl9JMRf-#$#7*pz+vCcB$W+_%Q#?&Jv0}x)Iw8 zV^>5vFN}?ZAicLdnEm`%h9&%i2Cd*c2vD2#S9GmJ3J73)s2jAYp@u>w6 zpz3PVnCrd9xT_vi!nsDIg|K5CA*kW>AOu&mdm#k#vLx>2nT)2$@#AV*&^73Dq{ga6 zWNec^mb;caSG0TXI!fcsbj3>tA2elKb92_i!OOw52(8lZ5&MKr41WkgvGF~XM`wH$ z8J@;ZM=6v)hX;^pFL{5C(G?di2p%Nlx<}D*b*Gf3Q@Zd4!i%4fdZ9040!7aTwfl4$YG=C0nMF*=OA68crS>W6S&fqElcI6tvZV=sT&;N5J8 zUYeKn#+>Dx;U=+t_j#M8Go6i>c$jwU*rLP2s=OexTkRyjP0SIpu}z8>N{KXp{(>C? zaf;psgbH?D0iPu=&spl$Qv;E{`T=Ml!YJfIjY2NxIUa@FwgID%%XqhO#^LEybnMCv z)T>{^DR0)8uF<6jb0cM^K1@}Ilo#`s^;`1bQQoSfX{dY&W|zdGA%06&kO0RDQv5C3 zKx=bYG$yVBTFfHk7kMsqHw^Km>kLE0c|)(pNm)Qhw0N9U^(4Y`$(IopH^8A>;RuBc z-s1@mq;xqqN}*iwP@`_S9>xsSa`ftHe6DSPudr9IPiGI^b6tGh_p^qtqtxDZwNotx z^Jz>e^;(wZ(^1+A^;)Pla&1+;Zc+l5VL1~>C;J#6f9@s-YCUVKT}AlyfivQn;e2a#Ss{Sh$e)pq26C-UL) zFiXE}HxZ8`i}q7?WiLh9*pj-1AXs%BE-K0ZPlo2*-oAu&Igu?aHe>mw_C&cwGDmw* z)64n6?E-w=8j%nvlDubdG?oTmTt1(6HqAiSZ9uP@ds7HB#19l+54hmi*GOJL}dnI@tC9rCt zJO8;IqrE@}4IRw)3(D9v(DO zC{jsX>YAy8>JZeOGH(+C$o^iWGC-`8yD5sPX+wTc*<-@N zO767ti&oqo|zhE4piR#IWD#7;FjP7i}Z!Ru-aR#awyxG^@sE81}mR?0-T4dFFh znV%#f9(pSwQs(-cj47`|x*%(3nBXKKc&jB?m1u0h{jIVAe>NL%e`|B)0rqQyp-M~n zeFX2;PIk5P`&-q{E80Bnl;>f(AtaO1tmom3VKHIa)fIu-r7bv?@4nXK zS~4>|BOR4`wpEKaHOE{HuOZ!!rYolkdj43x9-!BZ*=yOV9kW+XTOK@&sK>+UK)0j1 z&eFpQp(NNfxHDLdu&PFF&Af)I^OGdR`l5tLnWZmr3m8-W3=xvQHlqp7CFqQ#nabsx zm9>MhUWK<37E7pCN-)((EiqF@Tj*n zsL~U%GVQ;nCwxMd%QsGI=5C#)W(`FuuN~5U^SD+$9FZ268RsQ8DTpV$?T@H7`E< zNKcxk4WO=-dC*2p*O0arN`=1YEpbupxOA=DH6&E3wMK2e=xBtDobv8a+MH4UZ@U#; zO>LtE-OEIK6OxIR{PTp}YegA1s7%)i%6LCfkvPzJVGN-ne-cA^Kw+c-IH~FJ^w>D# zgK@^65{Bax*$a33{4_P`g_4#2UU*^4*?s~j(1AbuYA!tk{ z6G>e?Osv(_&-5@P6lYy4nV)H<9&-BX{Y-n**RxnV!f*pc_PvUy zQY>G^lZkazEU@7ee!%ktc&v)2DY>AE6<<)rPpW2B#ZT*DNGSTdR_cWvdPezWho03# zQz+x1xybwhQ|5LJYSS<{)R2}OzafXwHl25)Ep`X=pByN@mNuO*OWEI2Hg&FxDQ17y z9V#yal=?VcYY(3X_mcL@wSjqKysie?-;*OrQqQ8<|A&)U-+OC9TiLPo2lB8KVsHH> z=}(TNV<2fxYa0k!XYdm{KD{n#6p0;9IX-Pl4E=N64TM~}_0KuSCm~uk$0s46jTbsT zZAMz`fiaWEvsjSuhpNkpaob>J(&!sPK#)Y6BsHLYUCje8Q1+^U{%!q)K}t!Y9~Pt!U{iy80YG0pfO&droViQ)?xS6O5H z{|Xr2YSrK?q{Dc1+28G#U#qoG-JF_BH^$v&zp|=oAJz@LzTW6qUh+5a?yUK$Ai&uj z<1;?KI}#i7v=fhMo(9J}DZY>=l{KFJCXL;hw?28Sy-g|lymwsit_zmB;5`?-?}EP* z1YZ^W6Ng8_v#9-Q9Vqdfd4@fMXV`PARXKdzU1Qg+%U7nFxpQ94nW3FplDggQ{lzlY zs5cH~8M}Q7i)C8nx7G?1y<69R5GHM1pDKKCLeDg2`bg((_KbO({R3H=s_qKQX*k8+ zP?Gwh_Gjdx@C+^r(((CTF$AGjF{scFz_9D!Va=x|9u3pO%Qq;vt$cS-q`zMHI`1})P=d9Kk9(pf zVMP^DZyV_w_3$Xc@cg$=c+%bk>1pB~_3$Xc@O)x;xPCh&-R$+-US{G)tLsJAZ|_+T zj}i>er-mnYuX?7kux0oqdVH>qXieE$)qS$aaYJeJm|N97{k+w5ZhO@uP67Gmt?J*bhe-*BX#kis zwR5|zQe#^2Kq8rD*jsg)(N}MRZyB)FXt^U>qVMpsr2UgguyI_g^Cgg9oR5~c8epQ1 zUtBWYq2_gfrFV+U7ti#gseaNq#dVaL+x8Lb140G6a=C9p!Yq-OGFHpR?pmaWA0+pj$qJyebo9C+ z{`@$LIy#t?$#bH%c62<%iEQoIdVt;I`*}|CNv#<~f)e+qk$`CpmBz+G>MTmPEg*`6 zg{T61s_NIWu`&@>#Pu@Z`vRGu6wreVR18de0wvm#(MtQ<~M zM`>MHv8VpBG8tCJw?sz)HOFr_#VCwwVE% z#-_pPzz81Zheq^a&<6 z9LR|c4U-oSU)VBLKXATf@C$?NcZfLxr!#y^g+;F5M7gcl8~gL6BRbOUGo zQY55;Yi}*(U-3%bdHio}+#3lS#++}{CNv$Tj;<9tN?+!ye`W6)R0W*Ks-^sEk`jdC z=F==UNw>I?%L~R%{=4#6w~w*zAgnurRiL{D1y2ZrmzNj&PQjCD#E~jN9smw`Qad|$ zhEJ1HgRCkS17795nlP&TOa{KNIY}$N6ZZ(g4fP^VAI@w^D#d4QDL<>M@$oG&b6*p4 zyThE)VD34Bq^J>+(Eg3i@HeD(l=g^}tER$#tAakhN;e=DtkP2MCY5?nDo!wvwwDo5 z6##0tH@pdNs9fUhTb+?wHI}{n0wJZo&Hp}~;Y++Fv?sKTDgQ1VpCxTfi>11gN4{HA zEg;7WrA=T%!O(8^{`ouybcVkd2G4v#!npDu((yUWO6@2)&lMFmwl|7(egT*THdHS2 zRO!rP7%1_U#*|+n4>GXRIdrDxtAa*oPBr!oW*k{PQ*$9%aB|VE+~Mqx3xZrmbOy%Y zw%2$IW8oeN;G;UjKM|F1mSv@7)QmsZPhD|T`E`L)793W9c1&J|&L?HXHem^s-8>79 zWn^yGOo|>m5I|=k1mo8X{z3%i*R2XKWCv5v&6RbVS!VL}U?YHpWH%m=M|Gxb9gk|~ z`oa>?U(<*R?s&|l=az^V-OAe~xL~-vR2nL;uy~+q^Rxad3hycwTuAn-&glZD6c!Ly zNdczdD6gHP5WGP`(9t57rH0C$>Yq}`i(zGu+jQO?XMK`pN|6AnS5a?3II8?64^(UA z`${G}nb0l=c2?UQaim#xWCEr}`7t#r?}7OPv*mWv8N5S^?_vLLJ?xqEuv-6Yu8dd< zS&9UhJ!`6t4EvyogX`WUM!4l!qtJX`Zb-p#J@u!)$Nf}`R56(M%^j>FhZcE{pFk-M zAta|{hR{dv@r(qW_ju(D;F}oE*2CaXBzFz6t)xb?#-Tb*nt6OD)%&36D6P(r>OVY1D_kszoI91l(Ir#eR4R${QhIeg&J}oN zJ%;7%HqO#-TNA|?-YMwE{K3)k8EE-fw5(XAFa#!J(J3CDEWsdS;Z(hPg1$SQ{v;iR6b_ST<-J?6OM11n-uYce$Z6)lvojZ>nKg? zD76OTNs1&28Rs|@j;2AJz^?C;^s9-28T0iMDH=@Hvx^k-WsJ3LnQPEmnW274N(cr*NfrDRgY!YrrB8rPMa<~zXPH`0(ex4uA zKFfs0lTo>$=$^cI#GS2E^#hE}f*It9yh=qPr8*JOobI^c`%hBl247@mXC$bKbCWYc zNZv-3+jyl5rwoIzd z$84ps=@%IWZvwP1Mr1QeD;&j0q`svUV<_`txJ0c50mx(PDqQ za$Cl{Ul5j<)$L%Xvl|&!<%KY5oWu7A)AqHp*%XKX+JXPC)RWDQdD-mb*sE4ab;7-K z!i;D5s}j;K2x&8sZx}OVtTDPg(~)6^cIG6vq6hS~TC5CyViE2ZW1Y2N#aM)2mp9Gs zxc$u%))>Jz;B(Jlb7E8_EnZOu##OUd&`wfSa#Rlu(duaL3|Wytkl-o%rb|_6Vg-9e zpJm`mF^S))C*OVY^4%{7uO$m~;EKjTP2@^3vHMr0k&qE59bnFml8LF9szfSwpv_^V zUz)=#Fsts(Ky26#Hp5;OPRiyV!e;4@|R+G`aD7KYlJGZ&76UWqu%2KJ#H` zHCI=XOOfIBE|(5T%MTZKATcqRHu|x_grd{q0w!(ot(cbprhq$rq3!$OF}8ZYrb8>& zb_SC6w(JudH;}Bzv7Eu=zcrZL3Hb3~@?s4pzrwp4M^>&PTliJpW3lbRqZ6AV!(I77 zY*#DS{A#YRCAkzC?&fk`ORnJSJV!zhk3~|WA9M8%zM%}trG8Sfh?|-@fLn2Nd_<$~ zMk^XwCLBopHDQeha(e=%W*{f<0PSWVr$Xrp^8-0MYA4v6RIAr-M)NvBd17Ylg?>{~I1IJe^%D;*NU+kFGr`GCYzW5I-pthwvPE{fUGyKL}1& z@lzD4R*g!N8h6g-Hf;JPabne}G)cm|gC)3UDa;7+dz1>j7@i|eVt8KEIDJToK_0)I z=_+m-TRw{7D)B;G@70g<&=1Mr#RrU3E|nzDk@3eoaM>A;oKj*Q`G(K`;xIRNlY^O8Pp{0F_s4 zj?We;N|Tq{z=118LTh>`A*`H{g1MI_4ZfyI<$_rtg*OzPNFhPw#UVe!q)KXu@m$O2 z&N_Js%SJSBsaQpFSO%q+yIXiHg=3{Zjz_2TiVTnE2XVbkuByHZS4Q8XnEgqq`HK0% z+0ayqYc2j=5h@y^xEcBuuv7_(^MHK9dn7PGm1PTM#%%fW?;^(ctE#eiZhaab*%8WD)C2GZDG{HPY{G zF>`JP#IPC=x;8A0znS!&j{jO|dm6R8ZrrtD`Qj|j6dtY-UmKP#AXlvPLTkWr1nk5N zt4!5a!ly&GUo`ucsp5TQ_hl~!3kZ9L=84(+_b8P~uB^$@4ruO>s@!Sb0FRGruFS*h z87AiZWZc}gv#Av-T7fwMc?xH#s`{a_$E{4QPeeH{1aj;7`Z-TO=kg;p?F=vFDNurV zq#KVfT^L)wjJVE7Z`WbfZP= zfpCU`-!-u9Vg&<21t~t4O~nJ4MfQtajCjUS9vDU8KabnduWCF zpH7ti*9BXv_H9=>z7-_+sqM%6Q`@eguZI-g85%7yDOf{~Xy~n1@$M{BYw%%hvin4S zhWzlAPp-ApE(c07lG%X8*Wd=x=|M^qNjtq4g|=(PpQ`5*9{=2_cPJ^cZWu|0&7Y8t!n-E#vL5T z26YCvR+GH`ekk|PD|8K_ya=yY4SSrd$1|(C9Y`qPZ2q?hMy?|*pIJG1b+r5rQZRIj zTBPeZ5~AyKvN`U|=h(6{xQp0SYq*=&@E(4Wgt#>*AyQ@(HMpe?;fwXffzG*EEW*c> zBl1J)RxFDL_X;TYuzO)-I6lihRX7tb{zcEyua==lhHt)~F$459_-#^Lc=*s4JAfIFV&fq~}Q}z8*Uc-m@ zNfP4vRzjrARqOXKZ`9@vIF|*F@XSsz>PDrQx8F{kKKsT$BasuF$otjsmpohU;4z+@ z%v_PSM=jef;p0h`UlnKBHT-pw<%!}f^TQ`eh(UghxvhW)B4iU7P@42@&=-VDPlLZ8 z!t~Vi1p%KIs(`9wH$2gJ6jzSHQRSz2hncVQzk8d6#X(DQjMx*W2&Lg0@-#t4wPltc z8|Os=B4_bo{eJDv?UkvkNW~Ym|I&6}Wu$HzheN=P+|%1|I~{8 zsuRj2dJ#YYpS3>&d0BNI;0)Zp*iQ*0lx}pCzMJ7Hc(%3}0^Cn7Y0r_6k(Td@9y1A~ z_?Rj&Ag=?0=esYH5{xf%VDOvn%cLaj%N!K^mNH4hXU-7M7}y8FpN%vTG)+GZ<7T`l z_#!M7S$=`{P$_VGF#m@b$fT@UmgFrd&3GM)GpLGrCrnP`PBvTN3M;AGVW^vcqi$(a z5Q$<6C)OzeZt4^Kj^u3Rl@8AW@Eo}OFX!_gkjF>4N^zeJo%n}Syha2<;t2CtEj zAzlOcxY1hOPdn-4SvJX~yJ}K*C5~bRa(h_IRgR(u3|?e*B~V=yJhn64y?wJ<_UYq5cK1HEqD{!lfHh@YmX0wm5VmGPN^PZzGv`p@M#o z3~f`6(IL~G{{vpU{9&`AH+YZtbIds~*kov=I8A;~nh)gdufgF5EZU&`uZ*oUROIf@ z)4cNE$cW9c9Xy5FrE7bONS@E&*;dWY(nDjYybL_vCI~jl;8?1KAx>^@t}NBckiY8M z4!H{6A%8-&nqNEv5gZlr8rSI~gld*+uQQ+vlc%J0Q+w1p?KmkoaUJ z!OmgWfRE65p(^DXqvJyXIHPv}7{Hu1VQ$+$DH499aw-h8fpzZS#<*BO39>I<-9`U4%o*(nk7O=JR2yOmwTc(_>)Lqnm{X+~}0JX9v_l9PU$(pL&q zJzXp6ICS5rGxeG|4n2?-(J#6C=qq;uVOccdKR^-Z+6Yv54`BwoQ$rSab#!VWSv`GX zovjPM7#b)@}2 zc+hqzXAjCAxEY{rjAD#skaS^qPsYA6Heih~E!q4OfC_bzR>pw26`G6zvyD@xIF(04 zMPn!!y}`+{su|5eF;4Bf3g@G>INj1s2ItR(N;rSva0*6ma1M)c4vul!E-qlSTbtq2 zt?6WNYA4rlKH+c*1|8j|8O!~l|7z~Tf__xC@o5T+%V!9KCP3`nw0RW*&g0LG#ai7p z`ocx+{~pU?^jp~2f4lCo{a-qa*S1-Ml_)9eEC|YkaqHeD+eU1#ii%;A>lbsD7&O|k zZCjQ6;ZS~v_>CwQfV5p!hDecSmqk_DE)U~f%xa&PKwG-`u}Nx4)&oKUsdd@Pi zDbW!MWPDzf_G$%PyR*Rv;`d&8cHcy)?cAW5k`kpL^h%Vk3U9E6iXqDS#S&4zMuE09 z$sex84;0&uumGe*sSJ@KO_ZW4Ey}fd7qi+fCU~8oTTyyIHKNphGNKe6p+Lr>{HrL` zfR2%|Dw|% z&(^ug>QkFFgOSqPmTU_-B9&w;pG;}j{J^cE(FHIqan2dGPpgcPiV$g&mTZy6orVEM zu-_8_V~b$7+?a>`zH%FE70JQ2B30ObSMHz)cEdJ#*#Agit4I#E6{*73j<06JA$hx- z4d>DeX2X?_0M_ZnCA)`XGup7&Cba&N4vl^M)clmfZ@{Y90n2w=$?3Eo1KTY4fA!E9ia&ad zD6Ytu>q#-!pO8she=3Ul$#smuxNa@rT24RzuejEk7RI%XoPcZbC$3`^i@9Du=2}Nk zi0ejC)K9Ks494{a1zZmjj{iq>Jy^_YgwUji#!&pxTlbTr*!WZz5+)b0wc`InwpIeR zEgOsyKLbJq{a7~GP{m7?i7Qa)ox#CZl(5eW_>E0-NPKW8?~x)o^-_(mywbFjj$O@~ z1udgtNIGqzOKVV@QQFGs#5Lo@I5nHtKt1Cw1191iF? zt%va&*T59jz>&O1imX=yskD3zq!RNrkV?$gKsvE1s<;MHiTN5xCFW}&mDp1aOjQkt z{cH^!K@2r8&DDUt^}7ai`6O-NC_v9?{T{z@4a^`OT)Oa#U}J*d7*Ydeq4UCbS&K1# zv}LfaXjQeq$NU7S{`Tj}%$Nbt*o~bvhx#bt)mt z^^(mW)%R%ps*P>|YJxeUU=he&l|&*&S<>ImRkkEkq)ZI_X7S&40?!oQig;vXZ~d0h z)&N6B-v>0Va{R_J+J<;=b+}+#g5U?F2Fe1_SE5HWZ+}_nc1k~mq)3_Swfj_BzTKx1 z^X)#Bm~Z###9X^iCFa|GDly;gQ;E5D@A>{Rd`IFNmd)6n%rj!^B%Ug=w|-aAjsT;I zP6PCu)>HY7t7s<){S4kCMb;B~DlIScRAOG}sl>d{(}_8urxNo*PbKDso=VIKy{5a{ z8TJwNS0#lr5rEUmNCHXBudk@oS~Q}ebXc!eGX{TL=WdRn08I{=D~ zEfrPlb;Y6^*IwMDeg0YUKrK z$$Qe03y5qzUq9#ZBYkNbK^C+@LBH{C1Kx{;ceN_sH-z^hkFD8ZQ_yd0xlyA9OtRnsOfnRVqfHt5cu zrpwmOpVoFK#gMate$&zlE!T*ayQ;KY>}m0FqNQoV;`XP;@)h`#W9@8Z^DV#IQKF#V zlxzki_llCA8YS0AD%X2Te4Hpz%kx|4iOxUdC{fUFO6EezqoU;3M#(LrI>C$PZ_4t~ zGPaMX?)q!xLrCSL*vQQV`t2j z)`mp0<_C}&suZNC>Z@n_fhu0o0XK&U`67Onw4XH{v0OMEt9mH!*5~^KQ6=+(Pqfl# zXEqgTJCPTtQIG(cB}ggS`nsm=X!|U*9U-tM@e|W_KD4>fA8+ICG^N{aTvZFK#vh8Wv2TqPq4H3R4Vos?NdVz#^B=3&eagm-8 znvr4aS&sfxksA8trkXedHN0xhK#jNq`A+?Z0OWdbn6Rh*GNJQDj|OCPIzMSmIEr`E z1fP)71f?aKa0V|>qoCh)A41JB0E*7!XG#0T(#uQYx=-RfAvxQBmZL;LijuyzkSR7Q z9t|a*m6bcsC|N8@9#-X^!@J}X9LtbEq_`WhH1y+U=kbqC-JUojQ4dLRd=wT>*R6ej zv=4S(nA%*~P1a}fqQTQ2n;IOq48YiocwwsUFce7JY}j28cQJ_W(qCIonYpC>?$HIb z)gv-E9lo)Lca4^R@Li+l2J==;_)sC8=4~e#M|OGQ)wmj6!`^8m(IAJh><2tPR^LouEX}y!w_6NnILh&TF zZb|#kr#Dx&ma#*`_f8LXEY1#+pHCMO6*43pDzl5q=-B14!QRCMDe?zjWIjc7AFi)j z(hBaw1={$IQoxZ>9i@pG;cNxv?@kY9a(#n+^3_l|2#84**Y~0?HK$bt7obWcE{ea1 z2qv3K&VFw?8(#eCw0`ED9i=e5keu;}2^aC`K2}v^cria5U)**+AqGmpu7gtgLwB{f zKi!oK1yUi0RL%2$&#%O=8?kpRaE`fUg|fJI|sG0}1+$=#X|WZ|Tnz5Z2#+&glqar+qs^zwIRint!Zx2O zXAL=B(zSa^7B*xnsgumoMN;;%gHo~q2w632~6k10Raq^TW+=+1;=OlFI7oFhm#PH=?u*@SfWL@dmRvC zb-!LM`WoKdpl;Lf2A*S^b0d$=<|s0}i60${b~B+{#Zo!08rOOWt61@OuiM5Gj}ND- zi~Vpqtjr8=A!iKrRvsNxMTWQW!*OWAW@`ku^N2#cYu(3xaf;M)bLH>yiu5E)$(_<6 zs1qm73ue}o`-gX?{0GH!6!^PzJN8xE(|qa_iFi>ZIFtCeKt%$h zhnDh>dE)Ce2>W7iIdFr+_yPBclN%}n&}fatF4m0`a)4Qfhy(~K_5(hT;72O5`jpoNlI`Gvc&gz72cjk{gFErKK&m>RUJD)1}w# z+`!kh_tKHcu?{(IQ;ttebv}{mY-?N9*9+W$P(i<2!S=KQb-8m|?*p7u`yGM@c#agI zPKK`Ny7AK9>&8RESAr>OG|HhEWLwk4@!grZK-g6FCuHj=?F6;R5}?EKJSNj+cUgN+?E<%-K>#5%Y93-duRAFfM&`&6Pq)gj~Y`t zmu`<>s)XTj771goU|LUEzFFS@e+vUFvO?jM0sc;cObn(5#YrEqPus)FJ0MihZ~Zs` zi5ri<09sDs2g*=AMoK+H;yf_(NR~XD3BU%uKDGR`gQ!E%zn4($Ke#wbpOqcWXJyKz z%bD6la2QijIhNh)NSxZZa`S^%8e4wGfwRwv0BIlna9;b|wqKIK`w0}W*z-#!TVETl(W|a^ruApXL zB3b~?1|Y#Xg8RF4eo;!f0SPsOvuCZ^iP^!+K z&F_g9wHH99!V3x8#4^wNWX62iFV({eq4IVOR(t&ePxabs8k4@Qd1QKZV38?Tr@dA# z2V3}x=<2jHfL`U%jc*hgzQzy4*#|bkM#D0kwSdp&?=NDlvC1?XO~$L{x)}%0G37rB zI54gRlR|x@W6Bsau8(ev@oV(3La6*O{wIRq`@I^xdP9>8UjJglxz}0A*^A-aG_F3c zv>d_5Ts-Isd7od6I$1GsSIGalCB8yFU&jdc&n+4kS>JD;+H^*uQZa5|z!e9)i%I(e zhfrV81k>usExU*z_0fkmCJTKo!&v6efD1iLeKsT5`fA4Q3w!MJW9$pZO9^nCR8wYF zO3@!tG)b)5w1ka>yW4~#!cqo*fdsVg+G$#_xWnwhwbPm_*J+$Al5bbeE6FKv{!aR{ zr|mW1>S}?Ks5HcS27MG`w=Kk7 z#sj=3HvtSXD-+6E#t!cfDmyoNSr5bD!%FpcxMxQPG61m=)LW9CTX^xS5=igo;O&$p z@ySw~WZYNJk}PH5H>UhH5dS~g&I3-X;`;l$%ZeyQN>DUvSYWwug%y;xK57IO(Fk$X zMWY6dn%ELsjJ@TtEhyNqYwUtz$9`?m#2RC)F}9d!{%TAUTTG16_xn3@=RWs7&wZ9% z9^d_Z_L;d;&YU@O=FFLyGc)P{I`bk|hgS-JClJ35u$Z~)Z{%qVm3DS7b-mnzD!bk@ z0$R>%8~-yd@>#`Fj83okBA-`Dvar=j5sFk=ihLiZCp_w#)Q*=JPksOdx?C<#3OY=l zaO2CAtwG{LmnV1jnWjLwQkThBZIF0{LfBEEzhtPHD0S~>(M&VYy&3mswSELHBG#+Q zhWi?>N&L92Ugz5g_eXgeL#2msAL9gbsuuD))6QN%7ui9jAgW%_6s!0rfCT?i-d%f~ z_f7J4g#RIPq|ya45-53^T0g@_0XnRrYUt!iN0P22J)K7QUlJ>o4yMS}K@&C1=Y&@q zy?vR2RX%!W@>Oe?cPdbFLPHjrQOqh$t^bQrdpAbyFG4L(WLHWsv0@bomPS6M1K>qvvkwx6r@|T*sNi3BHeD#B2*pUEniYv%NP8(dAU7G zD;{VhvKkL~K;nV&&Y8(qjR!sjt6(KcaI0D>UV|t@Y?rHAcHkvknSAleRv)}h^HX5B z3O}&19Y4=1kSAX5A2wmh07Ok{2GNg4iqSZtE z`4XM|$jK9ln(=x@YDuM7Px9CzrbeSqMbJq(%u9 z`zlIQjKn{5THh&FBf_5bRk_B}{gm|UN?N7X0JSxzVt=I^L`rAEq&$j(Dwr^_gnJ?S z%_SlW7Z6-HULtfLIXWU`QHZS|Zn6enM15URxPx%RwO!a6aoC!;!8ZsK?-pf0J_QwV z!rl$HO(;|SB98>p;S1X%y2E7mU_l7W`qNz+O|&jI2{|7LzMzuxjtPTcyPMfqM@rHnWlrid9n!{k%a6{5 zIzmu=+mrw(xE#P30OTN$UE?jbkje%d2@?hzlPNV1HX8wz!6p#9jRJ72O5do#Cg3|B zVkVHR5W0fp)VzRL+}IUty|SfLFuig{%V@q^*5@ZKy`Z$^N^^P#Vk~LJuA3+!?xsR_ z0|N1HhToWGb9ov=r9}-KhvSAa71XVvYzhFvjYu62W#jR>&Y-|>0zZ^Dja0!z{J~{a zW{?+Z2FdOMHwS4733JT1>0%o>J0fLO2+cpgnQ?3q->TDe#o}%$_>A_G39mNVt8#v5 zEZ6ul`KpcfTd9zjV2YsC74z6cP0XLNnd@Wh-aIblSVH~VN|~qSJvy;WIvXgu&2Mt% zslE+}jwj|@1J}mRxEW{iU7Bc8@HPa{oFWg+Ifzj_*x2FvlFhPTWfWT6_7bDe%Nm7( zHF8tf2=J;B%u!^?&)TqQhx38;K^nk?8Z`dpA5c{=XQ#q|AN{ z6)N-eL@tJE`+T^OYCFNDrpY$m4Kclm?aT;$1}xTAi<=0gX+; z@7#%o-ww+92Q~U84#Q`x(SL7^{tS)&tZ(!kmGV!il+Rnr7d-G61YE5F@Ruh>m96_- z0O&13>!b)XYvh`R*&LuE%xvtOO6jpMW9*#KvN_)^-_p-!`q`A9Si0<{%CzzA#DQYE zB@Vt%zHkdt$KqfHUMCI|7;ecA%(RD~epPLf=@|Says`9ONPnoxK|F3tQPl=E zb<=7CTP9yM@pzaD*-n+&k)J3HFOiXdqHlJ#y^g^9ljN9s6x4WgqDJc1ZU5zPpoBXS z9(Vnn@w%?Bz;G9SsL+w5>WIV*Fju0ka1q$|2+~k~loH?`E2QQS7)HpteA7HWrT)%wSeOx0JmSTP2YoYgd6vu(gmNf^M;E zjL7jZB0s|ocT-?Tu*+Z#6jO=~uyhP!LDRU!ohJdO?xdz*50bg|2CnWbwpS>=PpG4W zz=XT=15!m*u%fS>K&VZJPn2in!k)_EQ96k*(dJ}jZC)?<0g-UUlcS}FJZ*0rM}a zTHJO|uRPyW`}2Vw3AzGQf)**S?i1WQM5M1P9=vAK$$UTEsPja-hw36Bwr9We`28L# zX9Zdbc#1y1St8<0Z{E^RQb2D0)je$w2(wo=qq)gkGjyPMFAkC2nvGVo0x$4*>z@GfxedAbo1MXCmQ} z@K=Tv0C1I+f`%{fHhpJQsS7TtGO4=6>*wo8KS$u!@4!zYeeu^>KuWTvjJ%4JQ5XDL zYIE+8u*hq+ix?3S>5M$OdZKP0yxJ+y*8*g>UTC$`gmObDCpA-Ev4aT3TYyS;+Jx#y zhXHYM+hPy>U|W_hJE>m2ZCR`CqW89C#ial!9PW5U&aUuI0&S|= zxd!ezLd<7J9BZ6&na;occ_QrGEiJMUIQ~7kfo{1nEYQ(MU8YC zn-aMM$P33~ZO<>~!ya2Wer5tgrRm(Wdl<++UOl03zN@mFB<)m{S2F=NfdH}>h%Pb$ zEENH;V42Z!0pH?=Tq;!T@g~`!!qV0B3X!nz@`pFg3zzUs+sjFcgb+(> zC&V-e@r(#@DXBBqw!ztowD;&A17-V{~7*+SO4`0+KpD zR(l<{rH)$Vj22lPpHUrIZf{isU26!)^WSSwXTkG5|GsK znRFdjY%JW7I%<73y20xByz0p6?2MKre79W3kCWry4Q|5k1~Y|rJ(%s}1~X;J8_Y}+ z>H@DRx|d<^hBV#OzNu5O9xVv?{KQ%ITuNg8mf)O{>ybw;8S-}S8W(rwUbXFH1)AD( z=H8j@y{_R|eJil515!!2Iv`%mdd;O_#H}oNm%@!eMa0Odxx?*2L%pMhx&>&9+a3_A z?{>wEmD|WZC$xN^`^?xo%Zz&#mbzNWNsxgVAE=!f_o0sOsg8GA9UoF19~Lv-&Uedg z{5WRZH@F+WV@8E`&5Zjw22hsnFk_l-Wz6W&f`AWIFr$x5Gowc?Va6YIjf-PO)pj|| zm`ak%j1>j{xNsLxVMaMMFk__b^^<6**YQP@n8_|~`>Pu26E)Op1geda2z7PMHM4WQ zOtdDNOD9Iks)j@%#-cfsqEDj7xYg@SWJ$@%iR@pNvoINip@N(Q%2d@LMU$RNQb&_k zX!3?=(rPq$Uo=@Jn*71j#7BrGs#2`NM@qn2HD|RH-s!{LW2_rHgyqzLVnNyKrLXUZ zFFKr`#clr)if2!X2eh?V*h(TyLhHB&XkIZ%-EdMY6{oI)RJ5~MM*sv;%1L3?Q|0*{ zD&Jl|7|iYy@s2Xgo>Y;~dd%|fPx#Bk^7|8fPCDn4lcYR%|NeyK0b_!ne{xtiGF0B5 zPyx<5fyT9@dJf589JNJW+~x02NaB@aLdA^*ndpfX(OSXrR1knoX}m61a4Hjby@C^b znr>&&RS60w5J8RQr0Ta~{eBo#v`U-|w781C-;2M}*OGM6t$gk(-Aa(^g1kEeDqu!g zP*r<=>Fhse_oYFK*Zk}U|k`d)AgllUBPm47e}VBl!!>qZ#H*?bJcWd&`wrV z97LU`CafR9Dtu7=2ALnuef=PCAEk?y7Y4myGd)raN-F=oH#)Wc5Y+v%DUg46M1(|5t-Gw}q6O7%yq z6o+#nUeh*vm0Qbr7feo4f%gTI$Di1CQbS#nDN5NG2|8QZ`70~Vf73>R%9GyX6h)U) z=Y*ykTqNKmcf!i2148zMm5@}SDVZmdwQ8UIY~o<6Tr(nH@e=ETLVOcICSR3(@{NOm zKu_o=Fv*xOmodTGdoFUw$O!EJOcGuNGf-SWR+>{z%}x0&n(`qv<+(shH|0XUT~lrz zo{v8k=NI60;#`5@h5T@}LU0kTGw)Hpcw$j$xPHN8nIl{Uz#A&g z9e(&BL(zSFg)vrpk$j)kpwS zKMfx>|DpH~3m(I-?&eJl-k6lPW#9{>~R)6qV?JBGU z8#5+h^;!;A$HiD3FRXqRK4n-Hi#p(FbjX{ukIX ziGKlp(f$mDJx^HN4PU_Px}gHY7hP&y?vQ)x$b? zyVaWFr9fnidra*2DtN_X+zT4xUgq14Y(-ri5$Rj?9*-(mH(ed=a^dqD!QdmO=JD3< zZ~z|@dVd09j1NL@M#~#~xBP*h7^%g=-Q63lC`mx1HTOolRsZX?qA$I^zNz&s;>F+B zDhuv)xUy&dGvVkVRqdH8;!O)KvuD-nd&^6iraneOWCliii? zs$8r!aOJzJMu*?LkdkEAK7{|L+H?dvrOY+E_3m=1Oz3o>SvQtFX8o?{k+Di#nIp1q z6Y8XZN)F%Q2UK*WL^q{?Z^#X}Af{*!jbyiHb>a2Ss!hexE)^$pmR;lKHTs~#d%B zZc*Y1@`VlLiz(3yucL$l!$y9nr7od}1icGN=yHlonkuiWMnYZ6K}mN1R><2amEM$r z5X?QLqRM;CLo_Iv1-nx!nY`6bspyW2U}Z{S{G8vs3ZBZvm$htdnL%=XbCQZdl0$Lv z;>)oZ$!BBeA3CD7xa~bmtA3PoMyv9`i_k^NaNvbs&S+)VXWW@Q-NX^hlS#f%777&f zv9~onlyVY^m(4%j1G3vfrivv3K%DMXy01y4^RU6gT{>8Nk#cPU`y~M2Y&o@a_A5Yz zv$Zfjqopquih1`{)pB*dBL&6^{J3xW#bt`K`!m8p#6>=V24J_QTCrBThQl#) zoyNDKrDW6k_yL~EHI$n3ov7)C z$lFmELguhZ5QGAix4S}t9x5hMpbCyuh&)ZLL-9%X{4e=&J{JnZ(rFQQ|C)pZw8cn! z(~e}i!y*{2WJ;SI3hXkLbg+~D3U2VaLOuDzW)W+)TC>0;_Q_*CK$!oIp30HPqe1S7SD2z_0%{G~? zv>@3aB^#q;N}JPU>EN6uTZ8-^kpMvR3=F3ws(pvIwh%AebXSJl!MFw04!^Fhuwz!b zqJo7trRC;3nKA|&;HNL&V|MNsGctbO!{$Y#g>2DzU9#k!?yj(HG*EQ*D<^A{K6~w2 zx{5lEzEq2mepf`r=*o*s-BNnq#F+VXXN7TPM*AB=XgN!cjftTPEkul=xM`Lg#8;uRxT^jHh2lypLVON0MTyEIm>&&i zJem47r&>Sa^$Ug~{NmB>?Jmacx0G z3R`(7yS6|#Y6SnRIJ#VUH-Vr*9quNefp7PRA8+UM-zB<-aGKDOxl&zld>J?VQ{14t z4-YhEQ0lsk^jnrux1-!-w`Ui1cSrDHPqDhmV|5c)C8uJ$Y>l!5v0%VqS2c$&1quGz zQw$=HK_t59wX-%ymuH99MdA^)c5?o6r6FUKq;4bqmZcUko^>}sZ2A>7(ezRiZQdPD z{ZyQ)cKcL}n!2UZ+N}gF)oz=AMX1gwq1su-O&@t~`l%Bw%p1y4E9Fk1rh8a|^>m#_ zreD0l@`#;deS_tZU^#{9)TNHM2B(xERVncN!n!fjG|CRQq)u_)-U_el+X@W7%?~Ws zMyg0c$dC7_nYdrK*h00C>2I(#3F-e2*dUDYFN5#{48jlkL3oBz%7E6Ve8^H_LhJV7 zA10xZ0!HJujdZ!WqwSW!vs0biA6?vvkJ-HoGr=MB@=$lWNFL5ooYwplYj^1A(iP_2 z3x|G(5cAB!A(mIDj za7N3vd}|^f9G8xTquv`Qtp3&oeXT5*s_K@=6e#nb;WDY z!h#*Z6spc0tKoOa&M%@QF(lM7+c= zCnu>aNMoG4RBmixcUKEJDJJNtJ)G-S6Ttgq5p%B+-XDkwe!zFjp8U9Vz=@SL*oVj@ zTh$sURap%thx-y3uW9dx*Nu`241dHAZ};TDjf6}>9h$R{N#r;z658}CJ24b5ZERB5 zn?RafPR#_H0>SJ8umgdX?lt@K?PxU(RXftB7_VH9j(k2*Op@Tz4QX1Z5pNM}9beAU zi{UJj(g`#Oxo{|A;)G_2i4&TWQW75Ds@r9YV5zdd9sDJO^nEC~rZU}!lAA%rQCf=& z8t^hvP6}9agJoJf+PZKM1fdb+)Z7Sqi`hiLAwWwv!ohsIM%a!Ul_Gt{jgYyuD)RY! zBP^N>1>NOIvon#7?!lJ*&@+&H#V^p3O|sjK5gxuHzZ+$E@P-F%x+*<56tt4^b`B2X zlit3p&k=aCJ06jeLS@r08uk0-{ibG6*0MfHCMdqFPi*Dp2DA41U4oZ01h~OB=nZPn z7_POT%2&q%B`0F$IFaCq1Pdb)oMiopBx{$McP;!Bc(k;fR7>}C*gs0yZP=fyOBjyj zb8*`l(rgxM*gu+YwRCVav1&$t(E8fRdp9f|Q5=^&Rp_&`vwX%lO>R4>(c-p~*%4Yv zy3e>1*DQ_;j#&X%j6NruG9`#M`t%mSIRfZjg!E;5X6y2VMrC3O=;Drz!0xz6=UR)7 zy}Uavigk^=MgcqNok)MrMI{i5?&6+{%A!Yo6-7HBT^F6OgJ>$tad+rFJ`a|zh_)K4 zMouY<)Onh8=IO+DyR}^6p0?H;im17@9FVR_R@qwne-%YLAj>KG<8%*JlO0RZOSDmV zy`3Vul8u*PsKKTPm}FJ=hCMxtUN$cHBp(JPUukDxuJhZbqqM5YL88m-DwcJ4!x&f+ zosK&SKjSkj@{>f=_ak;@sEA03`HEr$hP9qR05jfoc^x`9F_l8qsQ23^O+h~$rc3iq zQpI~cVg(SpWO3UKlR5#z_;}+aiTxWUpUh`g>h`!!dQhi(b-M$?6W&jPgQfpGp*;(<#3+I=dp6E z(~~b(>ZbD9&Xvul}`%2|eM*D@!SuL9SuMMhUSyYBI+oYrI0^l@DR z`sDUl%c;Xg3(`zL>N9M3g%#ZJI#9za`u~{VG#DkYQ>>Mx&=;~NShKHgwR8mY$(<4l z3-}CA7epTsiv>kQ3X@n+k&CA#@4B7AC&FQa_WH5G?exds^EB=mNq4(#yWEDP8;>0D z0G)v2Nkmz3&E?H((;()?RUAkEUNY${3Ng=_cxosCS+ZnO0ti{MWRe!TmSF5-=ayJ& zb0WnRX$e?u`(hkETWvdM!+5pr9PktAa#q_mL~MwPp-6++I8_c}W1~>M-4GioLDADW ztJ4W5YYHYhF9c$lpwI!Ra|w2A;S*#UqARx8xFa}^+$pv=pU?0DLG%$ZTPPw@nAk!^ zrr6>_K4FVOd;KQC(u6H?u_QO> z>U%Xs#qGn>WAR$k_S$t_L=2KM_W+>JB`RqFBK=}tCSR3Z*N#hQk;N1cJex*B;53Zg zEVXV48xW-8l{W2EXd#AO_US&O)()nd9oRKStl5dlZbN7xwNhvf6gpSP-B_Z~1zOd* zp6`|=`ngU&*Ye|*3w9ONZsxNZ)jXs;s%7$3q1tYd3hQk9XqY2g8lLoq9I~YsvA!P( zVP#M1Buq9XXZl{ac*$$`NOjCFtV$cD3)>?Sohk~`Y$>O(JBjTMn^TjD5QQxz}g-Na_Pdg%|)K?>BD6Pa|Bl}KS2J_faJhOL_X-?9|;wX z3YG`IF@b-yUfeo{*S3`39uibRvwNF0&K-C;J(h?C9sk0LUsUHohvgNbOVL~FFrLbZR=Gv7Qlasb_W!i)l;pPu$;N@$wCDo^DDqt6EbNA zEEmn0d{u;uojh|-JfuM-D?-^P9z+*dD1PEW2i2@6i}>_pYlB(^u^k?xH(#pW{1|}a z-puRjGg=fYP!$z)7or#1`@mY<8zK}dQoyk{?gm!4n zUdc#3Gr1+)a%E z>j<7pV8}&%rKVjcrK0s&a6#y}qiMK0duHAAIU?AoXd`|P=Q{mQq(LMMq z{FZO>>2R7^FkB^1V<@;tndfnWJE|gci8P$Hi!sKy1Z-?r@JjpH6O&G_Cx?S4sshPg zAB+qy>a1GQ2IVXylyX_fvxknYWpbAmIGt8a2|9&lWO@N;kq4Qjl%jGdbv;=VncNT> z{c4sFw*nOAyFx^IiE`5t;zhn=39*!9;=q>)s>Xpndmaa7@>S!&+f<14{WPeA0keHy z$d&rOPJ<=*3xO2ree>99t~BJdd=R+Y#`W?+r@2f(|DKfeeL1!F{riCm`R9xLuL3b9 zzh8KRVIB}sp#1B^R-?SnsfxQbDJT;g_h<4|qx?fcBlrVF1WVJX2;dy#pe>PKV14d9 z`rHL>TG4i%y}9u)nG1g;Z}=yEK>3$KS-{9XuyG$#TEaw2bOM7C5Ti)euGvkrK$Bj@(4v8ZNt&S%fVmMz_JZ$$I^U* zDqRm6w50Ib%gxpxe+Ptl{-!cLz*2)MxT^+L{+>1H^VAR{b`djF<*D=ws&qwzCNwK+ z(AerdIePXQKoCK4Y8OHA(X;c!UH=4PrTX!zkVOP7Bet3d@;N0j%5~>VzG@=qH42H< zk2I`Ac`j2wJ^&Xn(d){FtFvs@svi=Or+z5hHLFuU-shWqZ;;Q_k2mFM3XuM54`0Xi1YgfdSHe$ZKv(cTG5D~n0Gmatj^}kmBDktjPq2#6 z+t70VjhnQN$d#iW>9C#Q{FywBg$Cj50sS`y^m#d;6o+~}>wg9a#(Jn87;jiD`v1pjC*<9BNsd)0zGI)pjr&E)rcicqUQBGJKOF96Q6I|SzqJoZ z)Ih5$6K-F*nMcAnW03$tg~&Zjkt{f|Ahb#hQ788VQ3vua(W=y{M9}msVcC+Eo0rAm z;}p)kjNigaq$8is;@Md&ovPz3mf*%uVh>bEV<_Aam3?vcS$DS*mKhWA(=ibT>Y(D+ zpljsGM^8yDow=iUs{tpNlr+oB%`%aGg*CFyEv5`A?yP1R-?{Y^Sc!Yg^;%0~dePJ! zu#1F@%1bE165Q7a6D0j1?6v? z)pa&S37U0>1p|SLO+#mV!)pBIn@ZE-9=jy{@H#0g#KQjkP%|Cxvp%oWdNip>HM>_w zB^h}e@9V^geE#sB9oEwgMRr(^)6#-#iMq}GYs4U%DP5pP$gM0Ku3j@v8N?|^%GL6+ z?_<)gGiPZek{hNN@a!X@k8M{t~dMZcy!=y_P|ic%^onjKOvJ9(k~QM^u`&F>8S-~_a#c_?zAjz zFHp~3r7(nu&^NrjG)TLZ#Il~LqPdTe%JQB=;rGLx(+E!2ja zk`}u56?N@R#V_jG$A4|_f$R1eHLQ2v>Ggg4_UhMvEzZcVzW>11^`>pHdhh!B`mgn= z>(|~8QV91|(-z;+LEFvw!~jU!%D2c)bu+%D76em>l}PY*s<&dN`su{aCL*;{uV47= z7%$DUJI#x1Ut@#8n#Y$~^T1B^vx&jf6@Y~z@`W7^6GR(1>evsLo$6;3ot><Ufdo z6qPt(l*<2d?0ZTU)Zy6wB4^3~RoUwQXhcp4DR8p47r6 zUQR8{k~odlzP=WquuYn@I2BLmmvuNQ2QN@4Qm zyjRG@#;jw=mW=_MM{NiFPOcliTc$gznWuSbhZR-+5pEz(j9E($Y~1Y+jSL;Sv^grO7f=WhATE6j9@(2M&rRu;0q$vS->q}8VXFl zc2kD?>~1`Ok#c1?0y|~Q>hqN3IQeo#b1I)1h^BKo@m#I*q;fJCr1LrHx*{byq1p2i zyB2OKOe^ZxwQTAtXV;=7mD7x1)AT8c7iyX-7A$vr{m5W;nt?#(?f66rjcKoM zNg2ad#i-2kF$U^4QS%LAH$X-#nbbi+=Gh*PGl*T`QjkfpQLKO=ld2oVs#r_VW24xy zh#yfi6p0W&v*aLt#)b0jhWJSdoSquRIt6qh0>}7e2OyRS96cuRU4k8p_yi`;(_>v% zEHasEM1hcEk)8PrcM(({5wnOQB87=XRBVbxzQ-pl(#jFflwoXDjLKLf2HG`?6ri=x z3RM=g-cb&;-XxT-W;#!S&O4}I?h3T{MWwT1!OqhF!S@MvboL3-bY8B3?6i(xcOazb zya%7*o`UKlVmd1#QkdwhVpDYf0iV#ht-T(L+meB7Rg}t%WMim~(91QF{h=#t3ksZJ z6Gk(8-HlM5I?#M?E9QDiWJa>8N+|rWpjGPnB(*b=-8!_Kr3u-J^#>gh-C6ClFGa=e zbdIj++?(%~z4&p9wc7?`@mm|UuFSfgl!+YO#u*rUSQSlLMn8e~nhL_=hDi+HH`sLU zK2#c;eBCFFM!4|8PZKK4J9Z<@c7^>2Vu4yt&Ah6o9(BuPbO6v|nZWImGg^Mkx2yhl zwZ(A|pA0X#*1|rks_Z6(ARF)OXYy6q;@HV;Fr>ja&4Mv0yTKr&fbGoDcDB{}0||l{ za%v{V_aTN|h4mAl#l+w$tk}Y7chTXe#8jh$&m}rIg`-4=Oui~~_(8n2n1++gud-W< z8y^ZDbfzEjzgKV=A8sQsH*2>x0{3>CfVluFHUalZj2BdN{$2x(*Ol?YzKqWEn-3>@ znZq@`NSB=Nw~D@*%RjyVV+Jr33>()RyliO5@$g=*>(2B zGGibI#N*ICV<59z{7*z`Tif-h&a6wiDOM}W7K*a78_9htff^fZTh~8FaXNwGr+K964nLI( z@Jzlc{N${UPo;!Po=OCH6`m@}sJJ6sK;}pg^7*P7l&03x$wS@G5)|CCpjXCMitCE61jOUeJ-(8;b5Oxo2@SG*<*QlFSF(pL zh^$oXeEh~&dLskAQib6wWzF){g^FLvvW23o@zq5HCVVARV)4`2#CC_DM%h|yNg>GO ztHw{VI}R37LM1;X%g5FD=?e08gy)et5`=tyss^R0^?dR`u`2}ySBB3qKY3`rEvxg> z1;9||t79P7;JWM3%VB)%PwwhYbVersM5jrHMP#!*Az3~%PY~GID;Zs~t#+;A8w;v) zDEv(TTt}d3ipMLYA(T_IrdYfQsElIY(BLuDhG6-@?UwlkYb2*_QmB z@-&8m%jVo=I8Lj|MW$O?mAz--G^<=76`ED?9&iw$FQ@eG5W2i4X7W`dbfiMe&MJ*n z(tWbi6wx7>rWEfZs}cDMf+NL=lumVP(1}OB6qQ5g`^ma%gEiAkSl6gutMXNR7@)B3 z?qc0*C^u%^^QC}Y#dph<{5UPFC_`m2iY$#q;cULI1aIDg*X>a&Fuau? zDx?=N+|ZInJ1HhRv({d4g?2)deoe#Q zfv-CaU#>P~@>Og2-%(meB;J!<;GMXl$x8~w{XMShdg-Q0w18~aiy0z5u=cX5A@d*E z4>K~{1?bSje>H~xS}y!#D%TbK*9|^|5?~u`Wa4Y1jVMZ&HNw{1Wo4vGm)(^vJEP@p zAjjQPwz#q)yq6$ar>YfUMZBrwWmbf;ybbOHlCvUIF)57nmZjyr-`Onf{+Vef5AaNt>9__A>qrI{HPbnlPtl zbox`Ry|LJneDM!#Eoj=i;kr>LHf1a>tbR!;3v3mxsFyav2LTyd2>$}F6Eg}7AL56y z-zQb2sbmV1<(f)9tRjH4OptIt#5JRqU#dWv5yqyHHk9zG@U%rsyzuXTk_lw<8iV<%W@u5~g~8rWCj` z+?8QuiiTM$AYtMcR%LZ0yIvVarZRMC7+Htuj}fXtp2koJ$lS03ISkq!i;*)cU&>mE zEV(9Kslv-z9B!;IoHU!f$79%f0z(z=W3Y2<@_G(CZIkyzT+AwQF*5EApHyJ3fC3r%UjOl8G=gHc=(E@HOM##=>uu-#3Kp zfJH1>|AtQ!L+_Mj>2LA6;uIJ@!w;$SI!4gEUhpiRh|ecS%bNFTEm;)zzB}>!ptH zc?Et&EjLnRNefdm)q8xS|@k|2sGpG*H<&Xx4xS3_v>So zO;&WLWqn>(37MzbBNa-cSi|JsovBPHRNS43*jv_TRuGmI^g0C@Kg@~w;rHtsKiJzM zT(D`w%Ovd^EpA&ddhdouM*nwsc6br{0sIxabTWGfXN;eA*@E@U3Vw@%h0>tno{l)B zS@U>ZP(P&gj{p{Zy*0F+G70~rz^{ly{tT=vhp2DANvPuxl@-3l4@k%qJC8%mGP^Gc zWMDme)x!TNkH>rjVXA>ls@;vFeh{LGKk$nNcH?N8RAakODlzFum5d>3sIq6TLWnG} zO|!=^JrvK*?oDXuLp86E+%c6;2HCPpP4(ukMR2ERlFqZiFz6g#?G2qcn?4NuACKKT>%k+6e6eOAY#32kl|7IyDLsks<P9na&rOLlTK>Ve)a@if9lDOyyt{`6Pl{H$`ApXc+{Jz8 zKC{v^a>x5c8;)N%@T?8%Hqvi4dZn(T`%uCseIk`k=Z`P#Y3BOFgfJ&t3!P63F>4xB zTvLQ;{ebjzm=XL3;fMNAy<$WDr!{3(`yy@FOm6+e;!Uc{8=u1X2(RnD${&8rk1UO@ zb_GM0tL>o2l`zF{$_)NRg0v7A*V%bu6WwZ5_$e_e|IWDl{}a>1<@3Bs>t_~k<;xqF z|8KmmeB}@S!w*;fNhKRYw^Iex?ONh1_5RCuFol~nk%g=p7JsTe2Hy;z2}al15N=!$Wnu5RRu>aS0@_M8pj+B&|)wZX0F zSo-xBc2i>6HG+|n+c;>iI`rZzbD4VrK1I-3Gee~-i(;9OR~;INpvrPmmF%*2R_&O& z%FimX642s~3Hr7vANjB@`Xr`Ah29PyIVpU)BzjeRHUl5s;~4d+#HYRfTODCv!jniL z=6ZU?MDUd>>5Jz)jG~>Ls{x)Gg|rq*sWl+2Ltg(3J~MKUN_jgp*LtLZ`(lFL`tD> zk+Sq^T`Uo|+o$mXHHJ@=vAdhQq0`L#U76dP&6L$`06^R8JE)uaYOUK&)Xnf%1Gr^O z+FrkNN4OT@k&i5m4i!wXw&Ar4Ofe_;2L9kEwpHB`uU@Q(%nEb_J~&c=*w}DnuR);* zWp3gjL{GM@r4YB6-!`qzU2p?6cjciw04j}(Y-(MHZ%W#Y|J}oN_2HfzTMw@jbP5c= z$xnOz4}u}M<(G^lFcEmEMS&l7L`t2R%yT@ii7*P=i~qgt!l%5NaPgCCwsomU8GP!o zkB(pM%h9W$01IP?`vLJscD_3gBImpv#C3pZ6|)PZD=@5U8lz*77U^l_>P5LfwsA^@ zYCdydK`qLyZrYeSgc+u2$);3po9Wi1JiM0MX5822@$hXnjB==~?rJ+c7|w^S3dYyI z4Pt;#a%5ZNi0)IT-UuMrnfB|FI!DL!I7ak19(o8*=yAlBqJ012J)(T)v|*b54%4n4 zm$g}zP#kZ1^a*HZnJ$vjkV~n#$7<%Bfidnz1$AS4)pGT1Eh}rJ0|2g;ll$ z;V8ai9U#Q(x`qP7_4#39V8#Wb@tZJpZ}yy~ZkT$Jj5^yMe{gHxNTo;xqu<^p{cYu5 zQ;Hj*OGPSKGwvNaSvrXXQSamNbz^gIYNGi$-TXpR!!-X}OR67kt|tf+9Pnc4AaeIwzZcl$_`yQF4Tq~jdmivbKd>}$n^|6~I; z2vTf}PxF=&`Ub6dlL^$dG?JS@^_8x?iQ>{rD%lCurbPP*RRKR3%30wC12KV&CsY~4 zG_y-jsJ`Wjlang0rz)Exh#uk^qoI2$&=$Adsq0=js57HwbG}V0)l5sHpg-w&L!Lox zk{u(VJMCfEva9O_j<#3(&l-3BWB(gxiegXUR#A%Q2%tMMQ!KdUo4beRnEk3a8g1&K zW#|U2P=L0+%TEq<#?TW)nN~B45J*$){A6WfP|ZE0~kx9Cia%83fFPGK4p zgQ?`~+Q5X=7co^7PPKbaRb#a~jb2crPsblO`h#lpHomRVThg&q<<7}kVu-xCbHw1q z19M7FgNi~tFgJs*>{P^qhqQif$5ce=l9FB2RGW%yK?F6Hld9i}@%0d@XqDIsXmJ%m zKRL10O+N9t9a~rMi7`|+IXMfe8Ye@_)Y&her^JKR*vj&?*Ebcu?T9F6H`pdc$Q5A} zEl+d=Sj&qCN8M34@fmHu4_B646o5VjKBPzAAg<8ziI> z1lPF}q|6im{bH$B#uZzf-r#+}t(|DE8xzLAQI72m5&ebp-BFf=1J zXx?txjo1WB&*#rf)|T|R@%pJ7V%k;mq;ncA^@%tw6)ZDZAjIvvE}*>k2QbjYQ{%Ur zl9JiwdAmuwJS~ug_05_C`C#g>t2yE2}6+mpbyASJ+v^Dp!WL7;A*~Z;+jLWX$DBE$Xno> z(K4OimNtGIma~IbX_W`&oieqq_=>Pp{Jaysz61Y}lEwa6Q%O>oK|p)`4q#=zTCw^r zSZxK!87*7#>k!(7OF>TldR@QZb>XG`xw_)3LQF{#@~K6a^(lPY@R`#-zD`*{%K~ae zOUIbLV`$Cdw`Hb&w&uqnxO?y?!%Vw4uM06HN)U^b+PAAR2~XOUolk4mZ3t?w-vd<5 zS1YPN1l4T;IisbWUx(m6o`UZ>3cfM9uJ~smsw4@bqTm~o3)>k!`;{nY0kxvFKWNQ1 zv^w~0*`6PV+X2B}6XLxsoD`qn6e*$5f?#i`*U}MoAc*B62xh)oG5ZOaeHV~CYRs=g z>QGOxzn2kANfP8@g6(AZ99|-r1=Nbxk)ZWGLu(g)TXyEh;dWH;ewJX0PjHHqu#8}H z2x_lC8iJXxR?LnCv)uqWqh(iq9a6^y9|$Q#LtXJ*wW6X@Xb@5OzM*kEXqc~7G~9uo zj$jW%W%mRXI{*}Xlto9;W$5f_=-7!LP_nO@QPQy;sPjWZ=?5j0PSK*vCkaaLi9(9* zY&%Dy;2cDOD72SD$%Q-SjF6lUh)h zyq@izd(K4GcNN*tRb;QOBDI@)Yagk`?#1p*nOdoIkvRzU?kaNSvPgE|Gmy1NaPJ5R zQMMCr3JSaAIf>EXN}KnkC7wCjmgN?wvH}M6qyU$=>jKzs%OK@8vV%RvPYf(|#lOK% z_TCRV_r`?-XlpoHj;%=ewx22n=&rVvqc=E)`W6heQ~&7T5<~qBhH5#GZzl>imsaYd z#ajtIvifPHEK&{W04e=&iZ}Y9KF|`_Ezj^D-K)Cd#XUX zSDLb3N-Cfz+ZL8qyvCn6;NUqs@FUMyd9>I&vm-ndXu$S$-{;YDy~7Ql384RuQ7zy?ZA(yv7IQqJrq{ST=XEi!)!+WT4j|UkUFt{ zbPOODx6M@-{ktYrNAWGz3J&WpcJd3H|6qahljeW3l%Jji`pm3b%pIX!ye}|JUflLM zyLiW8nXIea{@olL-eXAg(9i8^Qo%D~9xrfQmk5n3l)GQuQ; zM@%V}D}pK$cP$~XSkIuK(@MY7(*6rp`{e%Haq~K#ZRQf(oWvARGIqBLI%9U_D63!- z&>!lP+{((>)<{LSm@+f?OTP?cO&}w+<3onV9W=ruYlP0;McH`W(j2Uo#l__SmphEu z%4fJtF=8j#5V{O%X(|=B1s=sM(_EY(jy8Xn;`X!R=05gTbFf#n43ze+m8NV~+5uLY za#2wG>e+(!cLm+4dR-K%phetsyOluQHz#Dk zs4+g!y<>|;98U+%T8WB4$bB{$fWBx%l8Oa`MI-zPoGk}r!y(hHBSr7qMhq(6{B`J* zx{u7@AfNUL_N&&;;aAg!V?7*sd!V$+{|R4-3Zs& zc4hA23c;pI{s_mc)>90O?Y7GsTZNs3*DXw{nDAtNbddvtdxnS79gCsf^^hM!g)}1h zr#v!p*spu} z8}suY8vM9>`B%!%e|Ye%)kKuK;^7)N!RUy>v8q@@C}-D|*%b1XLIs;UBBg?@h)xeX zAT;L!T2c~DL`Ww4C3V5+{gOBlP0H6l#=waukCdD_jSgg-cAEOn91;weS(xXrm7~^4 z;n&l>2&stRM;AQR@UsaAoo+@;C*K;xlL#T_W*w2@)B|16iEYmgA9vNjXNFrVZ$xv? z%w%%2kxZs~TBhqvZkCeC)Ue!4ZuXMNBp&R#BsZ(cWD?_boypC1GMU6}U1xH$piHLT z%gy9wN1043FE>+cIh$@LnT=IEODWCYH@q4%(R0c)|1_Z00Cf4#^c-nwa`LCDK+M)=S%yD0}IAPBSE5uCkXZ4;horw;aW+R32(pI$udIqs&}2D^(_X zQYLSrYLPcrPfF!2qfar>mAa&nU5QgBzJF$F$C*7p1ZwT>(?l zVLoy?jDT4zBKU1Yu=ddh72ie%OB0d>*I_ATh9Sfjwd~%-1>KrWJ<_u!MOk6ut7S=zDgzj@)@#{RT?5XX6hXMBWEc6(@cqAIHcr^{&w)z%$$zDqUF=qY75}mk>eyrOKsyg5S9=UIE;){>}R!2%;V|gfpZ3 znAd$BduN2SMd|cwL8!##*E)Zg~g-#u?vUgfLf|&y{&{6|FoolH?ik4!RB2prCO=n&V zHj(O+e3jNv5#=`%qc**sYRkO=w}N~(ibl+hi@JmeVkwgj%2%(NW@*CU*d&iwd}^S~i;$lH<;DxgS;(7XvAzOMcY^HRi(mNi3@UlDu+% zB{w*9^=y5TDF8k1VJ(uX=RK@N($)uJ0FT4~9>onGRA3qTA0{kIQ9Y3J3ql=5l_z|N zAF$B1Oj!&P#ZnY?_-82kI5~`^o{(qd!Y>8Q3ABOQ;wx^W3*$i7NjK`Wx@QIxJrq8;b@AzEAQ+QpIDlq&FKY)6fRKe5ugKGvP(A1b&xqhe3%NDW$ zcBcfpiYe_*GDC3}^t>XxB0*CKE947@BHe$>x0?A?#p1r66uDjPMg9R`M*cs_(-^8S z^|Ep#o{5q86Ul_co4Db#3e2hxan%+yja%G#QWHYwq^95nl4ZMy?)Z3)P)ARJ37_W& zxNia1LQVO4(gpsRKtu8!c^X3@!_w}lnf5zs0+G`D14b5TBOfD z`S+TIW}f^@Ktgl0t=CCWNpt1XEo-xTg|FasT|W$}ac%FHk#z_5lf7d69Da2-Zz(etL&j74JZzDB53&_4)9; zTT9di5Q1Av`2Bo%hQe8@{8!EQN#jRJRJ6zDua1sq%z>;RhhP$VoK) ztB*CB>cWgTu!5#vlEY}qt8AjF(1KI{hEv%E{%N6^E}#poV(R8yXf;Mwr#sB6n;N`N z-pXnSV%$0Yj@Q*lg@^y(hZ^bHC?UTrjy21o-QpT4u%bqqyr%B|Fy5(I=Bxrx8V2e2 z5oN}s7t(J=%LjbNGG|p1g`e;pk6!=6>uRLH@Kb)Mk?w6upooVW(MJEPjn!*M{D&kt znuAtlyykGNmIYl(^n+{tf?zP01GL!~q~c7j?LKE&Lc4DEoNPzz9T6X9NB?HwvD=OWG@kZI|xe)0#ul=TOxC%#^8@Kp76B~fr) ziQv@u8jZIy{JDnl)>zBVA^W`auM?UjaZYG90@{5-<;ROr5Zv)%yG{+bkPvUIn*AZs%^7=-2!9^Yu$~p=K3X3nOf&AYb%xMw=1-)t->rfbzPxl?GUtc zkWAb7&;q)JC|Cf2GD%&3wa!PARMN>_hz%P+F*v>JyyuF>n*R(EjT2r&|LjF>5Eusn zVl{hgHiJaA`>s@)g*B2>S^NXC<5h>f;;!`tT}vzdCQZ8ZY&`{;V}Fe#-eOADI7lJJ z6BB@L_fFeWvAsi;o^u}PhY8QwOR_vlPP$N)G$zuEu_n8E6Jnl~@MP+nBDzVWB$c&; zlTv;X)xuRb2{_+QS3+yIOEVN=tL}NbG#i6SPPy>KwPyR}MB%2Q>2i9eyCI-j&-|7k z>{c%DrYLk}s@_e}wbBs3aANPt_EFABC4rHV(AgKb&AJONX)CX9Gg3OTje9NBJ=7-> zP+Hq(*g`1&vtNcrE)RVn^4269YdYZTUXo~AlDQ+3HOt1@xQ-VSR-st2@m9)$`)!r` zewA|X_2m{Q*Z6X4lABud9eElD8xi}SWoX{vdo9Pr8*NC4CL zUZEU~JNGA0g1TL)`=&s9Vriv;`0W(Gn&Q>Abok(FxWVfDWF{ZmlP>H>D4NEOsYSO) zuqi81raV?C-8$8;XU9FUX@zdwo>`}7#|0E6o#ru3T@#f4ggK2xXoLg!0sjv0@~E3W z{xQOv@Y#ejtFy1#G%PBtP8)rl0vETPAwx&{>x`Cxd`r0s=1GwHSTy}J?24u`*&8W& zX)|aIzOtJ^2>=NK#2UYxi&nZ4Y>S$z-@+gwsGpqLH-mPDSXA@!{?XdNi#5ngM^MSZ ze7CI0Ppn4nLdp!@$g*X9)*(TX)utZRwp^7GsmFDU=mVvC(qP|HL;B*gFszLA$eC3? zBO&KW+F(*M{|-oHb%omAue!yYiqaA*Lk~8-p{Tg4s{F^k&gyC=`zStvw9zUUTWPf6 zvObZBmttkg-VK0-ZxBrT$*H~legH(;52-&4cyarokIiUl;=5%CKXLp0P$|J-FI8$m z|KO)uZuqjfu6VSh6RdqUTAa+I1{Kd%h#EozxHk^s6H&v@kk)3PMuMMKMdNT4v3Fe5 zKDdS9q$6x!{3iN-Bu`@~7>K?RI2s&0_9`J!-}48D{Q(dJDK!9XYwjk+UT2Wez>0b^REfLjfSP4#Q2Tlb~}rp*aA{(K$j`Xufcy zJiQB}g^n-bD8f|2(Qwl)>Y_IiLwX=>aFr0IynFlFr8LQozXD_j*lQ5kVh!u4z{>1GW!SDwaD zX<_3H1S-wLPWeFSJS>3 zNfTT(AcIbJiub0U;=QRUUK%ZdTcksfTws|F7bUpL@ z$CZbkrO*w_L(f*|M&+Rk3GFDqU#)dK$<$ouC>EEoxh^=D-@FUFF7Wfn!KmFqiu za6Vz^e5!%@Oou}ua2#-WtqH2kvbtp{-PN9i*Y^AUE)i;)Rm~T@Z3b4Oe2+6 zOH3up#5rCXC!JAiUn}iws-y2umjO648;eA965BwhR`GsO#rs7U?-w!|>9n)yqMDKU z&YR56F86g~6tvNyHZ6^&z6OYlf@k)RZ$m6DQiV1FC%3kT;?&txDDX|<#l%svebrnt zDyK?5D=C>WxU{~kg47|Vt6SpI>r^rArHYgwvH(;N1W`oi`X%9F4wJ zI)OGWzM8gpH@Qs!-y=_BDCk&#O_Rg*NI11CRSyc&j^Kj+(N@5Vw{k8LX=m`=vW0%8 z^Ai*KA(B(CU*ZI_2oBE9WIWIGDxK*#PkI$&!UD4a`4#EI?~p!Tm)!=ho8>4lZ0854 zj|=Ly_#>4_cT*DFP+>S2_Buno0eXQgfZ1>XUn&o$6?P zyY4Vwu%>m9N}xn*Ujhz|gjzaa{Nkxk4`&Ud?&;yW*-i8iQ=x`c-0W7z=G6AYXV%}T z@^3({@NFV!P&qX;Xos@bf1SphjZe3!jT;;sJ(#Il={7F;2NmlEX^x(~)NM!d_)Fb( zzz;rhRu~`95J;i1VFentfLd=@6n1ij$ywpT+8bQub;sS#ajC}*4$f8OpA2QR_=24( zdomnUd@iT*-y@H&{4Ne3IV+40Xb7Y#Keqx6tNaVD^1Hgi zx2_xPfq%T7(=OIEamlKDsA%i8*AL;stl$U421@R@l3#2P(O-r&Hux#z&|vh}-R&wz zOMg-hr9wqu(xpLY9oBTq!b=qcbrI)e6s2uq{i$ zQmzyQgOd_;6Gog~IEp~XD5qvJej76G0vS&Rm?PtNI)d%#Azc1joV^AjJgs@mB^4US zHvG32I#quCw$dGRu3T{_(erRB+kgx^wAb$#oJMS*vZ}Yice8Dv9Hll;*6y^ysji50 z8|-9-XWL+xY#WqvrQ6`_1l^F~gk+Lr`&cfJx6HL7uls=x2KfsprsaHo zTn!HBi1eX?ofsI>dNJQ1eIWk_1?w1!Q;Mg-Vv+JX6s5sjzix{9LipgqMI>mi|B0&k zQ>rOQRC7B`MK8h60Q}Ba4ZtV1%7X5lv4e_#8JvMDAs*~dk$&0Oc%O9{$@PHIJ7e`= z&7rcVHMKB)tmb5n>%Kj<{4F#U4NH7JHs@IJ&xxa=hw*>gsY7bvc@OM!LXmAY3^C zS=Dan@Dq?@M3UnmiHf0B)0}~Mcn?d1=z?iGnZcsdl2?SkX*TH_q ztn*G#%nlg^BjO&}n;sb)GI&hq^pjhq44&L71~1Mftu3w1!&A82;U%YL`EK`dgV$~e zUS|le=*YoH_|~f^LO7lo<9ZgZ?mxVma5`1q^_?jD@O*q-XC_^h`ws2>?F<3Nm(_lu zO1);F(*Rc%g|1Qgk_hTwPXDrlEy1M~{P09r-Yw4+f`1IvDbLkpT6PrfsT^v05oP(4 z>~_cn#uF8o@m?vXA(T@)*>#`-vhRgtydHopOZ0P{ey-KeV*M=Q$4Segj$MQ|@f}ZH zZ^rAUt_loq;Roce6`^kJ83}H4B#19nSGlYpB|D74tpt=)T1Op%r9?z=^3oFd1T8|9 zKD8#Ll`kv}HsVLDtn(4P3PNtF=PtN#2cO}c{3H?e{gggbM5M%gMJf_^SnFMQ#Fjc% zfpe4asgyfiwtS`3SwACn>`iL+2ZU6p@+jvPm2)?K$(%tRonBTmp>wO^@3Hu#W|8vW z%TE#!*Gv(S602qcYBjqLk3i|r2F`6?NzGcaHM9R4&%Q82WKmJPtB!jDXb|A zRHrh6hruIK*QB46IK9$#;65e(Wh$|CgGkwFYE@v<#uD$`PwG+}?9<%(2r0!2I-Vi- zK>>W6z~B%3AbwA6l}>Yx=`^Rd4k`{nG|md1%S~ZY?(0f`7ISK=w3y;A1h7(E%xf)| zm?{sMd0DGuW>#Rt)iDc%H-uK1RDMWh)W%trWBhV3EF0s~C?#C^FnF1q*XEl}s$dl_ zf@8GwrU>|{>aw`)3M29LM&c_V@hBtl-*QtJiPw)we1#+Nd&y(Pk-~gr^nby$x04ne-Sxg7LQJES-IkhVtI#z?yaR^Gse*qZl7B3MR ze8G45i!6S~ep%UKjqpp-iNdca3|9wZf{nUbTP2YWx&)V)0ym}Ds(R-saBq-Gx_rX#WBymeywE_J&?EY{7*QQp`TvY)&?^VIWriH|6jsJu9}1(mJsSxW89$1#{Ex64 zznu28a~Eo<&jq_N6cBfAtj_Gh{s;Al)lmj`*S_NUQ^Wf_8M;CHkYeLnSJbsPaZrH^ zdG{W;Zl6)ZdiR}P-?wkCe*M?t2twu4lOK|CD9k&1^=KuETikZba0a7Whr9b(k}!>{ zb+TP-<*dd7!)d9Thu5(idGqi>Z(-RG%Bi6%2@gGhxx?zw$~k!WFyYadk4GPehnyPm z7!DraT|HXG@K_{z+zCBec3fROJM2qvJY84~ubVC?F#H-nNiI*Zdt0%jx?y;<RjR_2U3jkFFn`!u(H-p(v~6GOW(a#HAT>#)*IxI zlv5*;(?OCWwriA-urZF(-&0}KB ze3-}VK6ToA$IxkS93#%VaZF(?hliXR@ldDz&FazGIe7Rm;UP-o;PDNIhnyPm5IxTB z7p-G>oHY`9oINt8hY$1gxL@?RbR_gRbEN2T=E%aj4i7ms;vsr0>=&(E zn+^{-HR2&5b}r_FhR4aG$9d4B<(z&d#D)?aON3!~okUPzIGmp(m#5eR5@HKRM(1ex zfWU!`u@GA@vd}~ZB!Zk8QI-%pzhBf0q{VI4gy#t5^Bl?n!7<7s@j8?h7>?p6$>mXw zi5s0)j`A5J3nRz?%5rK%c`7L1+%F1&6r+5l=Qk#vv)EMkF;(+}bZ{$wAVG zm6JFfF0CadaRrhaI3(rNh@{4erTwC@B_tC*@L^>XOVXAh@5YI74l_A5U^b+Gd;LH# zJJeZqMMw6Rr8FBum`gVqFKy_`kyArCUWt5QK(rB%7PsA_BL)#i3?yulFi(L8C4V2~ z@L${A<0OCY9#Ry&r6Xd>|X-=meU4{b%UR$a2HZ}xcjUQcdEb0?+A_P&w}h}gG=_A zyZ}i#+BrfJ{4p#{b5)U(s>X^oLazxjHpxsymW*0XBTRRg z%PDJw6~%s=wW2Ky`$vWSGt0sLkqYd6X0U&BdDzcz*vl!yennYm`?aDiOV~dT_AT43 zRfUBrarapr?iDPw-CAiDQsU0GS4Y7@uZ}8g<*Fhl#X>!;h6Xh$(cfG<`nFZ$67}Po z)f3iR+l2T`f@2{*3$GL63Jkx)Pm;@vonJ_aUKB>_tzD)>FA57=lK~+vCxvoP`R3!b z+w0c=olc>Q&v*(J2z^3CVN$}oZFdB<%Bhk=|LF_7Xi zcAaR)N(_RX2umW$&~VL_6PxC-NDzdtL;AaX!S!-dNcWWM{eb(#deP26THN-R(Qy5T zqq~;%?~X2)^*%Ea=P#p|BkOl@n9C`XmMe<=mfwuNXV{;jwVX5KwHzaIj=FI2m~!lW zX0ShH%yO`w0K+AF(VRwhQoUf=6 zK5B~gC}IC=uy6UWsY)YM;_kCL+$$R4!=`j2C^1TH3yttXOJPq}6**r)BgkGb`hivC zow4A7wP2U3bA^Rql9{T!Gj=)6@I!~YoEn?K&$f>Oe>d?x4j)Eayl=x#JmcQWl_w`f ztDg2dKjU61TI~(2#ck(LgjN?!jEnYRUP}K;bMC7rGHjhUQFHF|CKmQ_n8>LS6V17A z6DIrSVB*7siB^GgFxk&xBBw@7q_Eu~OnzjT%o9y6fhH}t6KG1?{zSw|+W~l;(x$-h zKz@>Zo^FpwX4C$Z^1m5q}uOMeHQZx4K8lr!7Oaf@zO;Xv(P(O) z0*x_FAR?ykiFh4-6&RkxPm<50{J0q7(G4TzFQf0H8y1RWfxdETL|pWJONgHgES7yeGaE9iTQLsL$TXlm384AA2{D>J%%k(9 z(0p?O1JMf;G!VTop)l8BBBw@7G!X47Oy=ca;=??_axgj7VIrqSOvH2F7bd3}CXb6I zuRxQQ-3T-9WU^;U=d>+|(a7;^cmGj^Xy{c-sEoiVDWWXcQ@m(u*zT^I>^5?7bV6r+_tLxu~&X?_KP@MPmuE#Pa=q z=giE`?%stbk3a66-I;UFoO9;PIa79LZJy=r5%!0p*&p!M6#4_^@@Fb1Oxnl^8*>+O za#qC2306xs&uYVP;$RaTRz%NNoUIHpX(NMF)pt0^pmdHf_&x5MMHFBM%~ zR>Bo@H;)%l_;|dK`>B56=ur(9q6Ro0CZ6bPx)8z#txF3*iw{&obUcW?%IRO~sbz-@ zd5vvNpC#TP3Kg$MiX48%bB05s4be=N6$=MP#9)Kh z$+uhY<+-wW{H<%#y*O9alH$wcJX2!H#&~pkW{ehRjmwI$EF}*UU9z#onH2-5StX9G z6rAFDJiZVw)y_6=IyT$@u|gx;T*x*rMq4Y#q<+c9c+o~t4~ju3$8m^*V-U(;9O4i; z?}>HudD25;@O&WT`kE430uN}L!@>@0xQ@X2kck5y(Snb^wV@-@5cYu~jP6{z1$0Rb zW?bTc`6k?}4UET$c1wS;4EKd|+oL(L7~ZreJd#6!m$nP!e8hNN+rCSAz_MbZ`Ifo^ zmdA9C=cCGuuLbj|K{CQGG5`us4eGba@8K!MQjPbWTNu;6g_$Y|s{_JaVt-5VD(79a zKT8U(4qteX4-f-sLc^`70;*-=gjyS_{S#FCyUwM%K-Z`iA2Ze=z0Hd{kZ&Lk#ro)F z+`n-)7%NVjSFBjAi9=3%vf3UEcdNWi9PHYVS3b-Enenm0q;z)ot+^>u>@vMAJR?y_NM6kA3=h7n)Ys>iL>_fO$OPALb$Cj7@J1)$Q!N!fR zU-2gdJB=cpM4+q{l(RLz=2>4|=soRa_1t zg(rwxf#bB;19DB<^+XWhv^zTU+48sN8lWb1kr?@kFmf$sn`ky-G?J&%T~@RW?xDA$2M5F7V~W!l5zCSq?WT z?_MpZm$W98FbjyaWMg@knqP`417)lv3e)o9VhqrPcWX{*BaztL#8(evcqYUPz6lDn)d56Dx(U+4yjV>L7TPTG<$Ja>4_g<%vMDbrLlPNR0mFQ)_%<#Rrf!)| z@RViwuhH_A5O{noZuwq*+VZK5Xv-)blAAUIt1&g$5fjE^Fko%DfdFcFgVN+fCT;ch zJf9aWhta^lK+UuqMgt@3KEl0eIUcK9*e3>(R{7op`V>g?wnQ#F78%GRnQ}wS!Awp2 zuzJQ1ulBfmE|Da`WvX3U{5NiC2IGl|bYE7BUg{Bu_flJs*-LrMu0xNY7=J7-S~3xT zVq965s7*96NzZgjODw8FEZhl*e1G6Ahxv*P?+1dgVM;!(!tq2h#!dWG$~>>P@EOR( zh{fj*Z;HR1LdeevX6mAFDfPbDbQX-Luk-YkBF{6t6gX!rwd7qi}E&s9w5{>^yL_43m_ zmpaJoxpFQs>y{=&UsYxB6k$6J zyU0!d)S0-Q$==_&^dsovV}yL9;z2%?@ICJ9zQdpPcFbgXgD77Nu9d!~Gg*tK2GG*@ zPOs>051BafBiMAF*IyGuu+xtRI+vOc_vva9q#9E1V+8+0@p@ap9tVm48 zAu?ZELh}0+JA(pUNNgzauUJiK0h2=s5|N?Ad&PBeneadvJSBz_m9{^;M~9?@z~g(? zA;mLGl)Sm|1A5%|=^x(}5BhEbce`@qi8MdbE8wHVu$yE&(TO7ELndDHt%+rEv1~z? zQV&qV3O?Vk`h>45b;Et#2KaL>hBtIEy^x|w)WujdHXMDx9mT@Zh!~9U)B$ucm&;cc z7vNye%YOzJV@O$*E*Ky##;3&&dq>f66_JO|7Yslr7o1r!02)p#PikmFz-AXhH?xHMV5B>@hZMLMMjz%KcQm(r%7ZPa^$c}vZD?~|ia zbeyB5;{Zmz_X(7g^Ca=!n<=AAyjEW$?|mKu2U`C8E~P;cYvzJ z;b&=Vm#$?)d($_-YuJud@*$J9dRa3@zOqYcM^G~gy^vmZ6z*eVWD5{+E37$0=Brsq ze!t=vP+%{QjgkKmt0^5ul4E2Nkumb8#a(cja1I$fW%-Iv+3S@Mc>L7!6<%hElH*Ac zj*hKSqlh}zQl1x5c-ME_EH%#H;33ju6h z2}=-BaX1{XI3U=GyoLJ^fn=FoB04W>i{o(xAbx$0HnKC)2+YC@#XOqobNcMhg{*v! zjsRA(THCH5fL_MxLeiFAHrA&ab<}apP77hU^W5cb0BraZspUf^ZEY^IM2a!s{-9=> z3uC~Mb(3*#noE{Q;o&@xbT^SD(gT3R5h#(%jzG&Ukz$nOk&X{SN+j75Es-9GTQ*N@ zi4>vABM=W&wt%ML`L8Hc$r5QAilZ!>o1T}&xUyp{?NPEsYB5=>iA;W$$A}Y9tS6do zrItu3IhII$GR8&wS*@WgOQcglE=J75glVr7)0QRDX@s;bs%mVuM=YR6u9P0B@UlcY zJxSS%T!8;d;J*_3xEFu?Rj$G9tcy>aq&?`UUu!w{EEkq+1%l<9F|V=&#R5j0p*jbZL+D@E|=SeOis?P?EiKD*mH4?IG(Wju0NkT`jKBZ z+oWMWXg~@k4rT;lJiodoGW`-V9kg!gc+gGBw0U?UvW-j&aH%peGdu}@{0aqS;-MXu z)4>fVpah&UX#;O|rTo_QN@symOXCtA-0=(#syGApb*JM`ojz8|dBbI_bCAO2>N1u^ zQ$Nts_;s=HMnnv=@mu9nqL0ewftTWB$tZa64(V(5 z!li?t2f>*Y1E6Eg#<7*M`{^@ZLc%R$oeQx-BbzQ`rDa1aEFHvF5Y&TWwJXPQh=XGg z%3mBpm$6tkJN}_Dc(aVfmhhLc&L(g^Wa7YKq)P_1(GcufOKH8*g`jPW;YM==2z<|c zP9K&kF_nv#~ zV*-)m)O8jg&^TOcD`^W5m#6D2KAv&94$l~^NGqdP#glaw_AhuY>1NR&VT~MuSr&L^ zz0%DfYJAmwbO=jvZybU?g<}DzG|xaCBJ))WG2DK|TS0*a8Cw>3Eml+7<0Y2`NJN$e zzAY}pWx|d#c*-*CZ)w&_2t0o4nYEW$qU1P3CvMgrK+>#bK_JbEG#%_$8%`DkZlN&w zkVzXndAlx9syweZw(x1U(jB0N7mWA}??;$nnl_^s2=7FZ84=xuOC1q0GrSvrpu&gw z*lu>1_u+ua`DcG|+D7qVo~;4Hyk|EaJpzlW0~-FR1f)ynL%EAg@u!aMc>KW_jP07c zmmUCJp40?sUJGG{%W+?KAO0v;;jiAIN!a@=duWoS0B&*M6X@_W8u}>V`s*TS4dSOG z%bW#~J~}E+31RxU4Y`*X|18IzB*z2mcMcx{n?~Ia<5Jbl%}` z@XO7vEsn0;Xq(;eM_-7YnKhx@Vks9uxx`k~1FP{ploFfM-LZ~w4G$scl_GIzeZclMatFx#6;gjIl*wa(ERC{7(_%!}7 z#rp~N6tCMUBg>u&t+J;$(bn11fJEKw31as6)6Zq_(-tu7^=85{HfeYQqQWwmh-KJ1 z*M?=#=lr5a=>^bk!NFt@;20ctQ}-P1>z>75hR;c#Z)PdLMsU8F@e!@?Ijf_zIvd$J zd%N={!w|P3rE{@4SdCSQitqE~^4q*P>0@=y&*TZEcUz>!59{>rNit_A2gMZCDoB zX|I*REUl?HrPUeM_Tigg(-_uUxKzVpX81P##snj27EN9-OH~;eR=jSfku1adZ5ij2 zqVA|T)wI&Eepfgr70_Bi{Yo3q1~8zp5o4ybL&F=$f+;Z(Q?f#@4O8O6*Ux*D-Un@K zN;u7@?p@rs+LSD^)7KiCQk>H2OlkA*L$GN~=_6dKDKRtr7=Kup`w6BLuiI%P%ajJU z%9P?nR~g4*K59srQ#{kAl%mdeTjZ4xImMHW7(FJ8n72j42grgcF%eU;6xN0*aWvGY zN9j}0Hl}2c^DHwON(*E0+HOU0N~^P?{^4if*jUl$xKt}*X7~mE%8Z6m%tsC>Ubj<5 zmKBv74W+0%Do!=6bVvgnrq|*6|fMHrDYqF4a1i8GeJmGS-o*GO~`i6iyk|I(|%ygUVS) zin^oXRI@7U$VcO#0S>cu*70A63+rGa)?tN#bu4eoh{o=5bn99A5wu$xens||XhW-wGh92+#;WPAi-~;d_ zL&`1&FAU)!@iu^AfwZ_7btm2i@UpnaVa|!g3|VmSHh>4fg{o{ETPdiUw*iEN+vCVX ztkB42-Uje!YvtIVSQv7CE~t1Lz=KeZ;}Ccoz=Kf!;t=L-0MxB}96r2#8vrEyJ&qiK z@iqVwM;?Z7L0hKO^7dX67LE`|y7w$qgSIh-%V7*^0=mc1BPvDfULdUt7>#|Gqj6=e zRo@0sM%y9rM%`^$(j45g)G=c1CfaFx+#CLMNz(#ER)l3i94+kUYfDHz-Ua}L)RN|} zVl|}&OfG4Xh%9N&uU{9Ji6u?HZnYDsJF%qc_TJ~$V@Xp90d9W1d9*p6S)$}v>aWL> zPDj>*R8u`IvnhdFc40HkkMs&GY>Hu;-UdL@hUW7ZHW&0N{S4I1!sgB9@LsdM z5pIAYPX@b_Fc*$DS_8{#ixN|HV**irzuI4ZdH!-dsRTHSDA=kOO` z)2O?EOI0^B!vXj!vuu*8l0HM1YU5HkwGevQM)U0aBq!pCo zOD1eZH?u9VIo+Kune;~%`Z^OCU|8`HS3cOx>-}qDS)80~{qxeMpw07_VG;HO=Q6|3 z^a02g4@$$~&8@em)lpiVJ@pRas5sSR+f!3+Lx1>b+0pu2Rx>%-4C2B%n22>)Y*@!JZFvWgTSQz?`*_3B5YWbp zRyZ=7foqTTn|;cT}8eR%Ies1ha0Z+=5ywsK*!7*oZMlvqQrc$bu;`5mT~4 z!<6o*TMLfn-3?3iplwXao>E@M(WHg3curNVTuWOWrPWzc|8OgCY^>;4xKt}*X1F!} z$~c--m5~+2rEto~vZ8X1CPm#*ajI#h6%BBhTL0=CO#{S*buba@u)-v*qb)UMK1yp?Jsii}jh#idHc%8!-P)1qhSMR_??K(jbZ3Y#mOj zdp2(vr6i10a<_J*+UyKSwKJ$2sUA#7wJVB@RAX_eQZX|ehrcpXJpifhEhCk!!%6kP z<_)_j2_u!W+$2&8oa_ zWfa59J>}5~>hY-qThhhw5}VZB>D1xR$b!#iB0k%y06zP@&D$8!IXK*f_O&ml8~ZvE zXNK18gZoySmPK~@Yh_?h>ncuZb*8m_I1y|b)0%`!H7#a_lkrz(c$lg(GOc*sP9s^S zRc?5gqVA|T)wI&I$_)=&K|MY^WFy9mX@`ah$bvC35o5AKuMK13JaiO|X$q(tW3t~* zDKkJ!i(~Pevewv+;*?frNBzSC!LhNUskl@-VrDoEe`N-UDdrJNFOj_p&S#}1mDZ}~;jp!idvmOR?==s|q>=UBdevPr|CpaQ)!aV$XR zN4Mp@3d+T2ApT0Z=7VxduFb=vkZt5T8kZ^;Gs9!>$Jc*UE`CGEayz);NR)wFCT$2n zc601%@^=zw;n^>KGv_USGv`Fy*PVbrMj_cA;|;gRPC<$seFey(N%zC{)P2RmvpzX= z{j-Pi`P0Qixj}aN(5_93dtrmj5VIR(rw>Id%ydkU8n+SdE5@?8M`gar16>HttQY{d zJ+g6ZrJ!y$$b^L3AUhR;g+@BvAWKV!&8^divLyudpjhq7aU9~{7=-c{htLf&)~y?4 zew}86jBVj>key86e8|L+hp`?vw2j8V(}@?d_09xkV+^O-5B%7?sHJ^597@!5nZ;=> z#IRLp_Jrts;hZcOLL;c9jyN1vh00^JU zdzNXl zkV-ye(pHt4mq#9;O0NPvW9OIA>9ycq{auspwOWA4szpBzk@;%QlHaem7!=?frl9Aok6YHj??mvb(Kmeylx+m(<#vh=D0jh8dt{oM&XWR%+k-jsWkhiV^QUvC=1QfnV4#QLRKdIaT~vys7r6l zY^zkN!$vj4%wZ<%H8ldL;YL!*hfLbqpgVvb8uVc{=v|;^8gwE1=)u@Kj+#jtU`w6vBCGZRV&DRu_`C zG?p9{;)uD!zsI4{GD2AH==-$?%26SAlUhDx;x*V$T^6&W#{aOfo&c?uM)}&_;^Anl z$8cZwDE{22kvAMQK7|ze(NUvCqu$xwmKO_s_$fJ?j|hA1Fpe7U8OBlL;$dBzw(Sj3 z4KX`vTs&+PM~xPX)M%~Ci?J-N0J>j14Ehk9Sup@Q@N67gskD;3)F>p}sPSnC78>bv z)adi0m18Khco;{Gf_hM_cI7w@ac~Sm`HMs7sF8K+sL`*}j2hV%{;2Uu0_Q^}j{FEV zUK@?Uj-gi4Af5+xV-Q!F&laIm%;zBGzkWGd`0vuffYLG4CTQSj4E2IC%fw@EjmA(W z;CG>cy%Mpv4D8jdzTY$)x#BdxU^R+!BzhCh+P+-50y)W7;QCC{XWWXmG3HWE$ zzk$m{z|Y_+E8w5Wpsj?!<(Xc<_cBYAy}9vh2K=+=BF_*Pc}4=aY`{GTt6&t`B3BDVSXQ>%IkWPs0n(J4554mH2KV~!k3$#o#zsY9) z$Tjm0g-=l?2k%Q8l=%pd9G`D&;P-gfxQ|cnMys_OsWsAqLayl_Q^;q?^%e37?r8&~ zU;`{0S24EmIa04#zrbzyCH`E2X;v1L2+U?>bx3QA|G~wWU^JR_$FYSkVhmnm8N7xJ z{s|u#h~&Icn)O{Y>y2zyL)JDczA)lj@=DZ^k^;@T6PlG}(yU}TRBpyZ%}RmsX0SA? zT(9}ko*gXwlOHnuAL!!KwEU3iJA4G@8{F4@jXyV7;0-iX{2nQ^qk{#DCS4Sv`hjBM zJ)ayx$wPxVSa@|X2MfywcWru==Oq|ocCfH~unZP_Y7D&(6dhkt1@M~72P0ey&a4;! zoqslttyJ8h3>Jih8!Y?)!9pXQ4ih;hjWXX(5w?1(K1$!nySwahVt_WN?)=SU8sfUkQQBbG^ZWmsz6h z&5h@Cuy7s+3+KpS;hY3+*}+1ZAL$(!EQo>YU|}7KlnEJ3>%8?9$dQhx(8ubPBafpLs5XxU1LfsAPR(IpqY1|Fl!gn_t5I7$)apd7{K3H4s z<}|ioZ%{V|@ma#%obH#Sg+GuM29&y+&tmSTk21@|v(>h^n2CVrKIU#L zKx9i|I1Z8d+8&bMuedQNq}DG&~*VEJi-Wo)3u$YcW*iYQ=D&oy1q2SPF*Zwl38Zm_W4VX?*70(bBj!b@3r zX}pll=@;Vb$b%^)4i`t#221PKeoW90LhuZ^k#%|e*9G`f+uE?iYc?Vakk%$M8n?~i zi`m=GYp5q1!A4+6t|G0jNu1?o;m%h;t6PIA#;yFw{S#a2!v!wSTKU5q)jqi;N3~CG zIVx66m9CS7B(ugOg#41#LS!SMCE9d{aF3)BKBTGqtuPWz41kNk;mWTzVO!s5aoG7>`CuQ&=9bPxBj80m7P4Lcwc*Z1REzUcN4FDgUAV%ltx zEwS3-XhY;-L*!AUg&mRU1c)pMz3#~^EeZOSZDhpLB8%yd5pky^qS}@q!txaIaaI-X zOi)(D>=F@qQCr*vR}kO{!_Si>y93o^VbPLD%A*YYDCd#TlPm+ss4vke7bxN21O3^HTVD$}tndX|UjcFHLL9SUTg1sSXU=U5L`ESgsEqB5kG zJ@pq3TV=cVFYXF>HtEYiXZi}#!lVQe%u4O~FHnMQ7SRLQdDXJ;1}`c@lEmWKSR^sY z%C*p&tZrNqa+Nwz{k{dnvEd(x$%>?DNk^M@cU*z9w+V#w9i$jgwAB9z`H3-&#ZC-j zwy}fz7S4MV6OGbh=S+pIprd#R%;iTYs-^MUExR@y&Tf_Xc-6ZUmi5I=a)e5ePKy}|vt z-H>TKOILsu{(^D`WP9(u{9M81L!r-C6b|(cvpym7xuLX_dWEBIBPvyr>R|iSZA4WUE%f0jR)zSzg&d2|}_ft@?9h zB054bF{1qln-8UmsQK0;ZgEEZ6`O7fs%&ZeWEfO%Hh1gx!+qUk{Av5)m}lNV1H}W8 zLUvt%vuL(4R}>5VuJp(u#Qjh9a!wh}8Sz)xtviLgb%vPjG+*g%ollKTy`t#&iYh=$ zeZ}26!I>2U;D8_-$5twOm-rwg+>CfC1PhIHIwSV^p^sU?p~_dXTPLUo#cEfM;}8eO zAe6s2gwBXrx6X+DI?arjZQ;*|4|naRKg1S%AATfm?P?oaRS*2fUIPxXy_WrbzjaNn4w62e!@F!Uk09 ze9$u%b^@F52;7?{l&3k4!o7X|6_@%n2Q$NC@Q0R?r#YH&T_S2r6(;Nz~Pjehk1d?TTiRiqjEuMfY)H;SDAl(^h1ZLrdBA@11!lyZwL{D=N z!0J|OGf#7{x{$QR&)cuV6C9CkE0 zS5*klLAGJ_TwE%v%nZ-NAD;^+kF*H;T=@K9lk;B<{dQ>lxo}G!&xQXml+T6FXSCX+ z;cQTWmYFyfMi9o+`A>Dj_POv8lH$KM+HVY9HckUU-vdW`YV{l)CWS0K}LVp-T)h;j!+T{Dh8$|2uz!_iyM zA5Ip}AI>44ENrzT;pw}}!|TGv?HwT=qi|5+9IbR*^-WK zmbRdP7*VsdB~?V-qb<#J^&%)}392qVlzF+X1brv^bg>rn#tlp zqYn&kxLQdkn~9icCc@fgQo&3!k<*ZD`OO3c#E6<^k}9HR`l-F0XF%}HtOu@!a&AJE zEse()(UZ==Ykq8HJvSLV&ERFz_Ba#U|z3a1PL@XVQO8QQ%wXUe^qm7?ybIMuAM2UVty2Vd77{xHB{ zY27WWc`NH?hzh^TMBd7>f{b8uc%F7mZR1Va#_gcq(s(ItV{x`^T#~X4iw6Z<`crIU zb(B_b8~+5y#y0N6rP>BF!@KZT#x_z_Mz#@`!YRYr#&xZ*jTCi9#i=ISHa3iG!?Myk z+qeUw!Zw(QZCGsB#-+Y({8ab;z;9e}4Mt{lm*Kze@BF(If9lmYDcplfOU!L< ze7sm<2`?(hc)q>cHbdME3-fgNZC^Ie379XtR z-KX=|KSRax?cUY+s%T+`;Cc_r0g(Me79b1m8X`qN9CE{Ar z&=mcsW@^q?&#-aT4Q!ZDbv}HoWd$g6^Y~T`*0eS(J_`tlRK%}d%pneC7-<++g8@EP zsh=Z3!STp#J1^*0d|nCt3WTZt1|5p8;I43xtcv8CZsjT4mN0ZZOTslV}CLQ4-+ES#_y@mov$HbHFr^ZQXH{t&c5Hiw%a~VLfA##Pm-AlHN862wK zw_fuogHdiwGXzps=XU-NwD(UCQk?o@7}A zN9)&MSx@;m8QIuuAR{5Gk*UrpggrDS2Y^6^~hsd~OV|n88R8S}bg$3^= z8*`q&WMj_rPfJ3n)Cs85lTftPWn6&wX`KP6hL0#gK4j9?rZ)TJc#!`~P-|(FrpAN( zBkMlLy^gJ#?n0%7mAL1_<{linnC_9^18pVt$O+Ej3CSSsSNtz7*sQTV^8X+cdC5I; z!g7<=AeRiW+TvG+NNkV%Yh*gM5^ZHkh&jj}$gR;H`8P>K&m7Tu3aNYK|05_XVs?p$ zyr?aHhbssW+arg~TF6$}tn82z!`iG?Y<9?5bI59<9dg?QlvP=7z4&U9vO{i^pWGp* zjy>hmt=bA9PQ1@s;dni@w%7#_Xs<7bgY-+WiVhC3Rh>{?-)B`;zsko-&_duKBO$91 z{eF$iGTg+n;zFsqzX4uD2U5w0Oxo(a59N|AJ`&xz^fORPtNSpNhZ~^CEKGF6r7lb` zGwhB(sOxWlD)hiLd%8y`-qX!IrRo0&-ILIW1*?TVg3!GHU19}{2PS2^2krBc;8k&% zhBq}4x;JSb9HV`&yzH}N0Zlx#O*|HL@h$DZp#!NleTtYywS!k40{hjt))?>F0l8Jbpvp3#$RUx zDBKM9rXLT)rS@ZHhIROZ0pjT%EJwAM0dlVC_C7j5maTOONhMN*Ds;0QrigLn2>v&0 zsszpj0f;4cZ43k#t5~psp`dNVf(?wU+XDAStV_u7eeF5r&NT(JW)?O#-r@HX8E0P# z$cAAk1ba-{s&zBLd9ZV-0n}O=FBu3Ey>g(=#Gp!<;QR_juH4$-JDlJw8VHYZr#Hd* zH9-l_>=+Ks(csh;e**|m_a`_GD|%p3rZ6N%+vD?6T9cW=u&R)RqAAR0F){8PXr}3n z-Jnq`UxI)1v_%7>DNGs)(~d<0IfYp?&`n`{D3vP#B~ut5ik<2mPGNjnGKJZS668ZB zZEZ{w1^?B#)Cg+n#*`?yJ&H{48Hr2nJ74-q^(CU<-SxX7KM@648;K>NpnLe| zZbm`HL?^r3M!|TdV+kO4k>|S-SjoqJ7z7G zR+Pk(Ft;5A&4*0d+DA>8gF*iZ)LI&^p~HnikF47r_kNiBGww}C-4mDEQJER;g+Ex@ zYQkJ)i4>s<7v|u5MzG&B?4blqn3KCUhB*^QPVQ2g2-@a#niHaLcuO3aj3QTVO+ns` zSugsR6z{L?CO65vGNgVN^Xtj4Rj`*}Bj!dEi z`H)FlZPdh(e{?BL1GRKxN*tMiB4eI2ajE9X%y1U|Y#f=5>%w(0LYG0Eu*$AYmqDHJ zI5LOO*S7*%;>cV;mskO)*_5qPK90;I?WHl=my;7{vTM`jY+2F+DH}(YM{(pJ)?{^& zXn3@!LP{Jtn2@ZBxvog#$e}DFDVD{^#ccX8K&0cy^7_M(pNJ!@jl{Cz$Z|#m#YB); z?!^%=)3F4Q`^fWs39Pbl#L5x%GKnMP$o4W-i^LH^$Fofwp-w_d9K{iIKC5cD#(3wm z1&KC@I-kUmG?Zx)aWmTgB#!ui+6J!eO0-2-gK#ZzMDZk!_;}hQBRpe8Nj!-o(+-FXgScoUT{EiS!mrg|?w9cfhS~p>C z*?OfjL9L~62~Q-x(Va-F)1|Xfs6g$+{3~@d!32|ppf_%uNtqo~H+-vKVE&{c5LrRFd z1VyIjT#8HWIm`_Ijz35!Q>DvrU3fW0=n<$B9j0s3C>{}D7xiSSbUC44X$7=|xGMl% zVg;aPQ?^R^5Vw%D--ywEl$=15U7JR;Wl0aDY>0a(3UMv0$?77}@Gx41ln{3%Az2l3 zU6F>kMJyvJmc_`$?0GRDpuWUZ>7n|ok)H^0tc}DHAQ6C|tjlvr`o>en^N4(?N zf`tB~jwj(Q4P}}`+>7=-32#22>fg0niFOD}5UwS>DV~HkA5VK@gl8-$i6`OhVhWlM znY6Wkn(zi=z7f<~8gHeeg)xt;y8-upc)JPrrjy=`OYNl03~#|7tZX&Yq?!^bLKQB& z!S9S753ul3%EhkataaD&S5f}XZ8j6$5%LL?=^lJ_ZO z4RJn{%9VhU5a&a&Q{BlB=hKo9x118>Lndu)OcUbpN#DmnE!~(B;+{Z}={-;4QhN_G z!>90PL)_E2F4V^e-Of*2GsNA_XQyHz?ioTiv;tZ}+_Qi#u>w%DDfL}G#63sa_| zQsN`qqz6(q#62E`xaV1uMU0vhVPj#4IHr&i;$9#mt75Jz(h&C&%Sei4F>*1RUIB=7 zhWoGz3{#GC6DodmYRk$z*-!p=y-|&tSFkw#a+8E|c9FgxMd`}#+0X{n3DU4Cq}{m;Di-YVoE?rR>oXcq%oy}WhBKirxlWmS$8EMpv1(q>FN4v zuA6IbNS_#U91K2u)eN-q?ZY`Hau zj`+uY#P7W)Ds8CNV;WakoBm@N&AxS}_Hma;UEoHkdOcdy?9s@SJHAK7I@zk(7575sbmQ#i7P%7ZFf22icd@8 zN>9p=51F*pQcYZulkqnKy>x?0T-g{!#z_0)QjL_E;V_*+0E|Ejsv!#I%SF|P*S6Ed@LQ!1#;Czo!w9xDJ$l}+z36LQH40`$C zD6XWT5DS(^N6>3r@u5_*1eC-TABwh{JkGvJ;z}cB$cIeYYN;l!$RX;ZKrh{(5?6La zkulPpaH&Sh%y10;Y+TtH*M&E!QjgRt{Hy@u$}6cYo?Qt4R;%D8uIvi<5-R~co6@?< z$Ca_9E}uH^xJi8EfD}Q@#+4OOTp7o@EMnS(6;k5LZiHlI_^Jwhv=(Vx*_~x1#q1cl zn05aV5b3zGqJ9tLC*lfgBeATwvVsvpG11*tcyYzcbSwenCGz}I0;_CXv2sNHP2vhU zvi(hEm$*Xcc(#cv)JaH*qqrj7&?-CC4Q)Z96{2n^aU~698YTJ^g~k;hP+P&ZUWpb7 zdl0@QuIL&25?6dY?VAyvv7;oO#Ffz$H6Jo*Yi~7iMT~kp=(XU}9Z(mHdSu<+xcB4A z1l*f0I}w-KWtkaH!e49R3MoPvF0R0*A+9VR+_0|_FmXlh)=pfJbL6LjditDsi7V4l zRI=sP94@Gb6-OTcXwnx*XA+d~a+tS|!!|cKwZ&P0NX8Y571_DO6=q8VBd+La=q9eP zs*r@DxU!4~n%hdzLdWA{2lCj!I{emwEl5ZJgU0ui_&|WXB$|do#PCl!KaKAc`A{lZ z0!reF4@KKeew9Gu$~4N551F*pQcYZu)AHwmUb;agt{jXaW2A@RQjL_E;i340CS)D# zFkBaIq)I(f&HR1@<4QGD8lSKHmGDbj1ut>saKM*X3Fz6BZBst3{EgJ*n-(57iLY6Z zB8b_zqUIv+8~q>avWN)-fFo8&i7Q7Cl9e&n6=__V&oYu?S&UrFx{m||l$cltTUm27 z@)L1|wUJm>TrnRWAf|LT|HA_=)3L-;@wqgKSfsTHPT8nprHFc)L>2NA?`;xQ2p!Kh zQH2@_DQ^^2q!U_Ir#hi6NVGxJ2_>qeq0&tvvlSmu+rR;qXpc~a@GVirxDP&N?&E3C zjPQ&VCGjMx97Iv`A(OWDRTEXjq)!Ar^AQG^^vJs7aqmZ!lW=c3?8&&)4$I8&6#T)~ zR=WyDicp4&D)4BCD)<`00wrLgirlSvRI$hZ6!E6&$hu+puN%t0E%B%FY*IKA6_%JQ zpCVuhFXWVj5`~=Z^n5&!FhZE)D17HoO3T1zTW_ z|6xIid7x7wOGs;r7vjPm|8tQCVb=RMFI>z_%d^E-OW8z@|G9+Yx=T2q#qmFv5}I^a ziD7!<;@^=`IG&$&pn|Z)RgYe>n4yZNJ8#NRxhO|+-$n6@=;8i|k(pC>*wlU?)4U0ez6IcIx7H|tPd|QH|k}XH8 z%WK@`V16k6Wp7#F8iEp@*)beCSA$bqycQ6k?r&r2_XaE~40`q78<620F=&a~V0fpT znBg5+30YAzyqkZ8CkU$e$esdp0Jk|aBt-9ca?5CVmxjW?&3)qoLdo#Xhf>KCP%^yp zp%_FS=5n-8OO_25Q-*xVq^%8p9)_NH2UR|5a3koM2A6MT+<n~@ZbU~71EKR|0x{H{&= zvpG!lb}aUaSlQ(uQ@DlL{3;9wlDVes;PCXL9v-nclO&QLsPCa)L3+1>o?|3!-&*rR zLY4?g=5cU6{j?UfuxQyTh?K}NSP(mvk%S^%3B88R0>sPp2vfYWITwX zk}bDdTfvoNInetJFJwGSP{QNO8u-Gzp-CL7#^BTzA3*`?_d|xmism2^5=e}C;4_q3 zlL-l|DkPyOBy5C(l3k^+xYO`42buAo#vMwZ8@8z72JQZpw?mnRf*bd~rJy0fhf>KB zP!bY+C^m8Oy9_$xe}FRNLndtv31-GGUjujo^wJF~GybPgWF`<#<5DLO%nYBw9~wqx z{LkXL(2FYdNS(~@V{pcQa%zY2Il}jD6}-&&p9g%2mC&D?l$KI{#{UAT%QrhbZW3SX zAVm=?P2b-w}#C^0eP zf2;mAxf z&-vfSz3H+a;8MFRGs6$@2WwmHoSzh-3^(V8Pa8qc*6^MZFmrynTl2WWIseZwDt!i% z+0ws3xutO_XS6ubc4Xbt$gFz`e=1qO!fX6O3p1JA1hL%{yv5;mp zH@?lUnmojBV7@bi?@hlmWK;}|rg`6%F|^77_>g`5pevi-sULD|^JfEi{-)*WD`k>N8=0gg@p{MC!lXQdj@LU>lQz$4@?)C*&7p&} znl$+-O@5Pnl;?*o3nO^e z-?iy@4AnSS33kJuz7|7fI8p;#{52Y>Dt;8^zJyNApai7T`3!59iEYILT6oNq}Yl%U5+-xxDnMjGJK1bGV zj{8goS%^*Q@(d0^AsA$`mKc;L7lwnL*^0;GLwIl)_n8c`kh|z!ABwzC5rtroNn05- z>jtvrvNhjrXS1@}LjrWc&BEK#xn%i%2T(VHVflV!-EVQ9sd*M+X`sAm{2qnSJd?I+ z-h?Mv;2HyZX2k~U%p>b|#C;}%ENpK4x;%qBqYw--X)A+&MrcG75U;sA=$R-0uZgc* zAhi#c=l~*q?x=o|^>9dhUTO+kRch4dFkZoZE*?=)}S! zA9wa8k|l!7{cDSR;FcR*?eunpWI6Mw_Nj9OmOq9w^Ht9mMAYv0rCx3%D+iqC0EL)Q)pH#EjW~@Y`+=ICark( zOk`t6k}`)BO_s!GBU2iKBN%wj=AjLLL>Z{eA$UHNJN2A9Rs@^2@0Y{UB$g+8w$F{9 zI%U7<(#w6tEkN9g4{+y(4&3spz%}ZP4Y53<*O@#%^YeUAGMjNl|Y>r1idY4 zlqsm^In=Y;=l0%x|4Fka<(exDUQm?c#X!#s(--(a zN+6dr1kTSaBaw=#4nIdN6+vYOtATun+}=BmpMBt@*|tIvB0%=ebCDNguv5NMMW@P6 zRn-;MU6fKqM?eP{|h$ajcU7pTsZXs7BQg7BLB0F?Ic~lV^uE5R|M-h}gHq}c~eniw;}%`>Vr=JGd3{_qsw6x-d7 za6tc;4K&{YHDv>x<$%si0sY-nmp`wgy2W!H@Yx*;aKux`?}xVaeET_>G=f_l?ky>x zWwxT*O+~jk;8OXDmSxfi9(A~nq=25W6+LMxdcpxeR=%RgGid}LI@}LZK=0a$J~0*j z%K?8>zM_vaX#}}W*5Ut{0{X^QRM|<}sKNmUoyxXRzEd)2SQQ)8kf1)5t!X671sWW4iW^5QWW;Hy1Lq7&&|SK zl8t?>!@edHD1W1o{2xa08;s=FJLIFXs=h9RsuBHWN8zRnK=jyun(FT`)!(kw=WjFl z%S?XBuyv~=+T1BOZTunLAVDqRmQ-EZ$Yjbec9)}ZXA0P<{-n;avW3S@Ifq{E;hXmo`T@{ zY~+_5^41y1FQ$+`w1U3ukXNLDoHkx{K(Ay2z3zZs%LaPW0qx#}W8=ASyH3(~MbbA? z*pAt^9rpMxxoPon*IU_W?>e*zDYSR8(Ymc`4P}FMlZG;QeKX1I;efhl1NCx1J+p!O zIH2CyKtFdteY1fY9niKZpj|BE+dH7`;y{R34(PWIXk;7+@y7w};DELS&=f}5Dr3Cc z1i|mJ5qENkJEnmCD0~L_G3(~0&p0@D92Q+FYcQ+W7_A2OQA-*+36Dpa-*o9&teDt(!Y|2G>{oS>izH3=gO9kG1SR=I|d) z0XdnTa6pe|13l${p3DY%#sNK@4fLD?dNu{*7=OV5Js$^x6>%V# z=YCdC-#DPJQ$V*{pl=P3nACR;<9~4sFz7ITzMgsq!PwB;1VO*`Vy*%h&)Yh-aX`O` z1A*IrTcB+nP~CdD39~1}Mx`UN@wRh#jd37w;E0cOKts~Rw~yn2sc$S(n{Hq|;U*j8 z_8!T_$GLVyv{A09@@I@t15ngBV;)vX{eg4=xvAxfSs2uI$_M$)P&i`tep9E+5pufL z*CgvB`GF|gbrSNimLO|@bJ6pMJ4S{Qb*5zk%<8JjmdK9tha%-~3lydK8-m;n{!ri6 zuD-1RxAW8)6Q)d`Jj2!s(W^!DhU{AF%hp=Vt~JR28iiR}`3=faS{dcI+<~ZRt^5wz zN#>y!W&XD)%uv}#6s1+RJ#w>EwzFeq#}r{GOxt2~Hi;m=3ktL3{{zZW^6%xU+=HlT z`TvCM3=#i`qHJL%ILi9~E~!G-{(EQB|4TOgAU_d>S&~jcSxVAbj-Tm7O-nir*-18C zOHV~%h8hk;QMMZ1Fx85$d)x8wCXgDVg}qrUuG*}dEW%W5+N}eeLPbT`ts~c^D+-z) z{GwaGT!m~>)OO2FFE=Oe-!0}s!9F6p_=^GEa+Bvxhfl09KA~woOJu{r{9}gz)4k12 zWxqsax{WiiTiKPMx{SKe_lG##!I?l+4My5}BW+Qo4Tb_)w_>(PNy$}adcR?-l1AhR zer0%|W-InGb?j;C_>(IP@_S^`3M&5s4kjHkcVt>~kaB{Pa^JX=h#PLLb+QARlnpe+ z0qvg&RMD*Zsy@=BN0{_*lOBRJA5>Hw)J+yjs^)c*g_5ed-DIJpVm8XL{S(YWWokv! zRr0x0W=#Lxr129c%^o{<#>{au4xBXIx@j2D;j#iUEe(pr`SP$+BiLEO!Px9u;V2TzBAITOr6F-Hl8CJCC1@1@&BWHG06n+(x;kd5l2$ zW2A70xwdFs79Gk-o^+0&>aPHRVieJp232#IFN~0)S8#AQ-Jb|#F~6ybM$7Ft*@7$c zFZIBn7C19FUgZ@zl{2?fsIs0Im({np(;bylKxNw8xNm}Nx(ai0x7-wwIYIffIAu?Z zXLZYspUowE=R$P2aYnb?w!4iSx&5wtkKJL9?Ky-oqme_P-+cbeZn=HuO`Z((9!Whl z<)99#_;6|;ee)W%--;#1ju)Bxt2E8Gs6FQwsy*kgPhJNY4r>0tE-<^UtRrNH%wj;4WBUOj~Tp2HO)V4RQHfk z-GfGT4>;A`m!Z0Qp`IBNC;3yylSL7CXXx#&3TBlQcJ~vooUP#F6K2b+7l$EbYCXp}p5KwD(S?_TC32D=<99>1>?*d&rb+Z==yOIMwocnRVMA zcgyYPu;IoYp}(p647oemMmH3`F$(@Xq2N=1(Wza~?!OWt>?VjolA(y`9=6@CyNulO z)!lWIqN=-YQdD-=*qhVxiXV)Izjqq`R<$^8#)zp?#yd;2!Uo@D=kSZ_qZ!U7ACwI@y*?-2tJ8<$D`9{IG z*a7XYK-VRJqO_^1PXB+fIeZykKMM=E{H0Ljf04&(7S z2BdTtxgOqfNrkxyf_#sp_U(>)(9Qu>$ALh05rY)R#It(nY-74frE27b(V}6fMR;nAm6QrtcOpTu-&AYN&61+oqFUZ%$PP4+nBS*%;Z?t`BwBYB`_Bt zbLbU3(&}tFM##3zVNL6i!`xu@_+2MWPS@JCN7_pT`OXo!G4tj^o&-_9HwIOEP+wE0 zs?0KW>X_HD(;s0?K~;SZjC&@`!>INTI-K!hOi;CtrxmXnk*n%Ae5O)BaF$;M30c`{6~d zi(Qy9TKk+xaG3-8dkV-twGy;ApoMWDbk=)JXRUfnJ5$XgnpQomX~jb({{c;_@9!Z? z2i41a$kIXeeLZyW^iKm{GU-hwy}_i{Xc{aAGum0NTwP~f6z3Kupxg%eB|UOer%acJ z9^F(yJKB?Ich6?DkKIA=eP@rKeW(e{Y&4lpF+c_RTWtB1*sw?`-e~Je^lh-C0JoLx zh}Su^ST{6Sem6{C2UT}jjty15_r}M=ZG8^lqviez}b=bDrNC#i4i|UG+mC9}Pe~h_( zY0T{lV{RW9vwGj8?`fKUw}+08-sz#^qqlqLLmY28v%`_;<)wZ#;)oI(j)DLOf|r%} z%?yb}hEyjHsy^&dZn*d_Q+TWCaB|Mj&tXerwElU*0(3B` zqjflwZ(XndCdH_I{44mu0ev3_f@}E1xCT6CZ3k+(p0N>u2ULA(PYL$bCu>@DVo&iC zRVVZmKT&mjPxTYcnqP5@=2sl8`S~MzYQxR%sSVfEQyMORgvtLulm9o(uR2`osX5ft zcZj*4XYS{kbe2h{oAdyaPB!TTlkTl)&0oy@UMBscNynOWjHVSk898=TIf8YdB=u`J z=$8wjty%P{N`sbJq0$Ai$_RGVi(VB;z%nGY&*mdY_Vx0MUN!MHhpSP;z{B7@bJ%nl zKh?M>=Tsl0-R6JK(it;HbFI%V*vt#^JM_dF&+PGsBJBEOntK?v^P_s|lUPn1$6PSCV5wJ5iafj*tSfg^9pxJH@IV9Lr?jL6+&99lI>s;Oc*IB%uv+8Sw zkA^V{O}{3;pyH^WxhY=LcB%|27xc_t2CY0+4FX!2aNx+jr4;w(n*4@%aJiH zb*c)gj_a8_z^|^_A85hfo$b8TSt*OPPmoqFGrjm0liqB4^HS5BZ!})%AL^B=7OVHE zTBP2m>MHd<6;~Skg{J%pQ+~NAzs%6RSkvGF=N-<~K9@a3IX}gHz|{lnNnDE(nPQh8 z#a!6aZQ1z4^!(MxPcCN!`AaP^H(5c?rj5mhBW!*RWpWi%+<-=?|{S{)>)Z=5-Yy9MH>~n@%-1L9kmd^C*`8TxhRYbb>rp z8&GqgGQ%NGPXXB{hl4o|Xm%V3ajvVh0VcH{_R>l1dzx0iZPGVP{%a0_A{z8*pZqiFldWoj%Tx9MqH0gOJJy+9;b4+>$QcliKH}@xL zT78_kKh~s2AjM`;#sBGja4?jw@pCSk7V$H?ha41d9kjEyPLN;RYZa4te28)m9%{>& z#2?mcwUhY6dzop(l<~f$>ns3ogwC9>zs|WwVkyAu3JP#YuiOlo3XGnzZ>sP##m-ng z!19x}kG{c@hh}sFbPNbMmR;_l5m}4jh&4xL35wmY8XDyP=IJ6a zF>*9knAX^B`jolyG8sLFD?J$^V{t(jPA2O1V4|w~Wp;LSuX6tD9z&eEPP(ule~RdH z2QKK8X^fPMs}cX?joE{$%hbVubToLWx*#H(U{&0m221_dnLj6Fey*LMNmW7BEj0C2 ztiRmQYgOwncX$TvHJ~(k)zV(8Tz|O+l4P&HTo2#X(Oig6 ztj*jQW3Al!%Rijl{+Ot$r0QBL>o3wOLCx}P9m=jp2U)=B*r}79TLVe#Vq?dk<8!O1rBkQQow~?x-A?QImKjvN4kg+KK)*@Zm}jtvai6*H_3xB} zv@fj$|8YQH#(@yyHZd`-dfnbSm+PWwMQ3xrj!A1wemhNTDm1Oi_tpmkg5LUIK(4pU z;i`V@rTIVf(){mx>88bZ#`k~im|DpeUX}0vD$X^`Qn?QD-@;>!pRj*&xlw&%$KIGk zCwEeVd{uA%;ZASVw-YG@5XO$Upv6L&XcS|smgz%od6Y~Jh1hz_KVgUae?gt$a70Q;U)aEb%M$-Dl(5yalcj^C~@wtJaLFEQzPNI4!k zSMP%}KupIYr}kcrO`aaF4J>JGLH_FACgzuk7D2wHHy?MH$Cuar2qu0ue-;|z5Ujn= z9l!6?Nq_LXfEx57H9fqayWJFd(7+h4BA?@zrjal1T`r>j9b!zHGk5z#CY9y(Y~glJ zn%bFG$Yhc4%=~M#^bM}5u1_g%v}vl~CI_@M4utj`XMt{YK(}N9<=5+@i%aYF(Z!`M zeROeY9g|;U^4n>CMMWQRfHgTyJN&5Ku)`194Lf|V-LU3sHJ0kHO!{9P_C94R6I`qlS^C>?q!`pv>#R>a6dJg(MC%Ywts&qL2dZH>1K)HPc z9em!j3`Xyai<++arLykfZm-fZ?A{eo-G5izGqhv3jqtj8{-0jCv4~{NzK;m1o{VJh zo->b)roHix$=4ja{o$7!CwH>gjvd{I?YcTzQ6(~4nzI>=mkXdj9C!4?KM2!`~D1u(R{_XeY` z#9>^5=R&qIq<(EkZEZ+xcoXn?AYmb4O2^E-RnWUZ<|yfc}n#<%Hbzcn6or17BJs|O9XL;bqTG@?&#*1Q>bm&`v;p>Y*VQF}qEM6r zj5(9Gojra++&rfB$<3HKX?nU~W`=~e$C(w!jJ0m`$0pY}N2EyUd%r<`e%kbad9=ek zDiiaVvY5v?%wsb#kB?!_gn8ep`m4A_^;dDTv4f>1|3;Jl56!Q-UfZMMI%Cz>s#RB9 z(?_Nf`9-EZt}^X$rJ>hi+VcveT)4a3nDb?7&iPXf-YEv}WYdl(K`QOxC-lkTdBQp4 z_mhPuvv*pJ_1_>^kSXxRI*Rbe51t1N@)!2u7JsxSW!8>_Z2k;*t#S_vUjQx6m^yD7 zqO7jkY=je?^5-k5Sl-or5PWIztK+)1qehC{W+Jw~ylTYijiU*Ui#;(-T zn>6|S>3vr5%yN*w2-?BuDPz==uyze?{MReaSQ!HGLw~-{*u_1@F77to?w`&oZ%e4! z%pm7th#dsWB9*%5xaUYo$*MJxe@AAU-w7||>5wWX%R%M6tByiJMPEDJ&G$_P#szkb zzsdnsW&^czKri&k$>N?$i-W(ON0X2{Bfg&KRh><09Y^Y=IHo@9T$7F2!C}6PA)#02 zLQb=T_Su-79OkQWOl^{m*_d4%=Ie3HOQp`v*_i7&%=eQjdoM0{Um>+_HmRREQXj@K zZxPJ(voSYtm>b5+Z53KU_cg0MmeD0r+^0Ahgf%VKs#mvRqUZG=XdX`Q{_MO)v5As zePyZ~i~|vM_q+7hE{%D=S1gz{ldmSgdWpi@VeC|qYFhLV)1n8P7Cp$d=$s7x=B`8P21h9Y5N;YdbOqS$9>fmT3&Uz zrWKc&^zSCU)THNYT6K;|&o=pIneO=G3o@GZ z)XZkRsILtF_2s{4$`a(?@e1VJzJw3B1my?XWe1*em%kXDh;xiIt<#0p?D2DF%r5i9 zNsxcLFZSN%;H{?^y)!^>#`Fm|8)Z7Sl4tnO>}ZdWkrzi`{P|{Q7bTdF`AM+O_0Wr5 z+1LojNUd`nGN#WRF~`17mtGKCr$wCH?72HkH!oxTKWpCsCRMS7yM4~g>;el4m;6uQA7|01x3Y$U0@M*aRZ70*Myj}m~+mGIp>^3%sF4P-fIr;ud1$|K4Wy(plwuNN1JnAsb5ubehZjV_o`p`}>RhM*p>K(NDH5 z`q8#UKRB2FHp6MZPdY7DqNSTP(O>8m?Xkb&u)TN!E&4f0w{<1u!2c`LiJ7Ql50}J1 zOn+E}y&;>DgM6wosl$RPMx0MdG8PXDra19Xta3`??j~=Fu|toFX|*k~p`WNbe~SgY z89=L^H}We=DK1teaVq(@lutrr+B3nr$ez|k_OLE8-nz&*p~qw8x3IbW-Nc4(EWgnP zHvalH{(2HWTGxiJW5d^$aI;nq?e$mhL9LkAz@*yE)p|(v3bmHG3qcP@W5K#4*P<4dxckE2nuhnUBU;rF6x!mt445VL2jh*u&fK_uPJW zxJ^T2t>4@@s_Uku;K~;+s2IGRXp7|pYyRV{`HvIx=Z_V7{uujvr2TETzlY$LT84vq z$Y5DWTA%Pjr_dW1*TjaC~>mt9qtEu0y_LQfW}$02e`9p{EPP5EOI{gYSKZlrS%D2WTA_qAKGI1K#C=PzlSVYzhxcyP3y>SSVw-{ z30<5a)I{%~G4qsNmpiHVtkIppC|H{-JWcbt_>cL_CGbg);JVGHCF!1E44f;n&Eto!pV2OP8<7YvK4oeu-C6PiJx5(|Fub!o&fqWGmT%$rV;g& zC9H~`nvK6T8-8>5SEhK=lLjeX5hSuuZN+zV;;V?CXB+zFB-*6Pp6*P$U6xe~;d=$y z;!Ssml|~w`sx|W20cT^XBxK`ddZPOBC`x=iaj*c+uyRkCT&&bnCKoIAl$Ed*oNU~& zPstYTsGxADYfZ^xehvAJSMMofj@9spJtppP-=6H|rY8i-LJ^ka0|JdfIs50CPIi_i9o8lIdSbTbR>?Rj` zhQ{_S+n61*)y3RD(}FG8(GPUd+tQXgAUoFCHWqc-(c9Mb-?Fa%rnr9eh7Et+hA*~N z;x(7#D`JI3GHM8CC(euG4%_xMlG&>JV6=)Q*ex@@wNIR9(M80HsefS*gE z?#&j54=mkFy2zz^Nf+6rmz=BYWWzh!@D4V-%7$0k@PdTLQ7>5!{M+W~50|IkdSVrJ zCLbU2CRx97*Ii#b)AOD`c>rc2(<|X!GW1RyEx?r0KZYk~>T(Dyy(O22u;$6Z^RR!R zh@F#`q~cjL@9fI%882@$yPW)*WqWA>c0+2d&868gW|>~;0f3Z&pA5WqONf$T&lV*S z@W>pGw$1LAi8m0(xfnxw(UZ2DY)q#f?4CGtjCUM6v3ki*0#T)8wmlr%sB&!QI<|94 zG{iG+EGu_&FCHl6ZtBGYrQD6ZMHA2SBhJiD_mli_QF|H z_mYZhaem`5#>L3FeZPfPHa(w`Cg-9ey@YmQFFAfnF5<@&$>$ei%5m9}_?MShp_WH> z;xi_`2J*dgC>WUFxWo;qKpgXkZnNIB(B=7-UcpB1){GS4K6kpfcVsf<-W6l#-VtNx z-WFqbc~io>ydmLTUYGFd#iCfp*X-{i`K@|I3bX2EDa@*uq%iYO2|fR${Kk*lLVpZ= z*j9}nwT1qmE%XO$q2F)YlKUi{xfhv{cJZ!WS&i{M8QC<^L&4_YVT88Llt+Nv*&weD zVy)qkRN0`op77e68hedL2IBm$seBF!qA6kjvFG^t`bOo`y(;Qv_-)lgw5N}oCodw# z@bt-5+EjWKmCr@c#AFvpc86tm2MJ^NzEKS~-?Rg6lpiZzl^QRLrPtX&u*$oB3nu6A#4?GI(v13`2^85%;%=N1`N3OUmSSfu^o8F%|8UoL?a5 zc3P|%nynf3w`ORv+27CpPPM<2?e9eUJ3)TUcxRs7$?r-=e7lhW0z)HO$eCV~>c30e z(L|Fx1KJwoJpeVOC?WGe>TK|&pKx1D$9SSPGy=YvWI(wO3KNBa)$*CMv_xtu`rtX} z8oS67P_q@Z(otw58lmFog`zp#GYZ%hE>}1ncjitKQ|C^wzsK3%`SKedV?BSK_57o) z=eJw>Ve)I{x}rKbqfm@Fw8WF8R+{L@vITKC^4~UdP;ERvkDg_mD*V*uEZTK}d{AwG1l?}hrhF@XB zFPCt0nKQ`63ON ztrsc&OG^x2?G2Gp?lfgh{Rd;;XG`~9mzle3f&=1vGUTJ1dC3Z1~eQ{3#p$q_z4JF0sdoU9Yo+>_$CWzC0euDvyWR=6bU&VfKfD#Zg>P zA=APXi=1{ZXZByOCXIxX1yWLlxZ{7NG%A0R%vb&>nXmjoGGF*!8lS><()bjA9=|2O=1rHK#WmT3#W#`} zLKo<{9v`qb`FCLs8N9oOneTt&g|-h;{VLxh(?gr&vA7le8Mk<}+=XsQ=M?SquzB%T z4!!pFO{&3DF&Ndu*}L_Yg_y3rWg%wi-f|GC8izqJQje?bZ_fT!*xx^`IevpVWOVXN zu>1Wvqot?rx3YJ#jR-?@IP}Mf!^s}+~&9UiV2vs~}=cvYU^O(&aF z2TH6jiA3wzqF&n;^;%NY(V8}V4I93?gqzi%lnj-7!+iLZRdFU@+X0>ln3~?n+vo{m zFDhfbjqWXC$dYYOw~wMM6N6FNT7O3tUr%w*>d zlS)kS1$!a@OB%1m<^vR z;qgJ%7iQbvX8YS{f2Yf@nd-{Ao@`Jub*jr~i>U3MnXI{icJnBy`N1XdwnUQ=hNhMQ>vs?pq-z z`>;MiI$$=;8b0OXK9L#saTk|1bt!Ilj{BU8`%Grsr(N7+x^1he&Mm7|OdMHzU_xcS zs?-buj^lfMnw%bi>ir3WG+T4WuQEs3w zz47R50TY~n@udlm%Z@k6#hXY>ZIhA;O1k@G$Di)vPX*H`Z=cARDF_*yus}URM$h*0 zMfWM$0Jk_fH-RT;_8UPML&xBpRp`@n9Q*J!)a-NDV6z`Zg^Gc5#r{}!Ku|JQ@-;&g z*L)Z6Xo^>2cKP<*QP~1cb^=Z)P5AiicxSnIrxR1!?2_(j+3|08@fWU9f#-r6>aa!U z?9yp@+TFd?QZ`Ca*kTVm0S{&hc%YnsC!B!C)~Fx_c5a=PcmZ6Lc+?Ub{XKmmu@=ub zAy1VP@?=>dN3W@cf8?4KgJ{iC>Re4lH)D=K5EhQq1Xx#JS`)w8#-lFp8e(g1ZlD~< z=PMB#4sx^O!5_E#fAq#FC~0GRtyM9G2fCUFn$hl@h>7~FWm~u6g(V1D-No%&Chi)+ zxa%OU94A=Iqr2B0)?BM%2;OVL;d;J^RP>2uHQO6F0qcbXth+=38$0bc3~0Z+&G{xO z{3x;~RmHBz`Q|R}re)%85sbTyQ+z9rF4erNFKE6Yi;?C7~b^=C*1njg#0lPZwM+dZD z)#iMR3ZGA9PUDI0$ocLr?$|PM#|7g~aEkBY(PgaC-I(!NiucqqpXdba9TKqD5(P|h z+V30CUhDZ}6`rW)(_G?HJi4o^^;wC#x;n!pK0T1QR<)TbJWCHNL(H6a21|#xOpz|qddBEnj^Ck zcTRJhOMHGH@jmKXCp)$i%dwr|*iI|QcCORi={lQy8fG!XqW7q1nOUJJG zDxQqayF0e7<=9qmY|E8n>*2IsMYS%DPUT&lm9sP-tSa?&LV9Njsac|swVjYPvxKa% zL?HwFsSW%0s~B60eF>U!<}l_#<=KvdWa8n1X~q+e=qJ}bcI?OJn9MLIW=KCg$BxO1 z<2Pn-zu*TDOtgK!irMzl2zDiOC|L;0P#ABWJU}%&aX`i3+8Hw@;oBNR<-C$NMcH0$ zdWPdaZ9v6nJObb`4F{7kr~WHB=QueFz)xqK+h%P~Uoje7+c>#?hH6qVFJ}$aj&CeN z`dLYCFZB^jKl5Luet}EQQo6>&j7DK`tMq4X`XAV!~p~Wur10;5KBG77MN#Rk8Dox{BhMQz~9`t4;3~o8HYf zy&GM6*N4)(Zb|9gl#$*&Hod!TdUx6M?r`bd7D{j7lG3{~BfW=hdJoz39<=G*@6x+> zRK>u$I$VwQ>YG~afefinS*cH2sZUs`k2$H2l#%*)hSZm=)EBMP7p&ChoYZGRwt0FK z+H(3}Z6E{BXC(E8P3m=<)MA^|BA3)Fp`>0eldIRlNtt(@$hSvT494+vyd;q@wzL;- zjj9-W;NWSsjm5_ou_h(7H?vCL^|42f)9P{LI6P-N$t|8ihU+wrbxg;AsdW}V;b(UU zq|=y}9r+{|`2;Xcp-uf%pKv@2oy@!aR;qLc$7M@B)k!@$o9Rr)bo#i8nN7Z)LwHuUfb*SzbF-N)a!eOwGc9&Zi^f&p6UlDSz{{%ZP0$itdqQc2JcgSw z3=}qTY#RpHj#aj;9NXprwiA`DzhmncU~`EMb!>wIY%b9ej%|kkn@e<8$F}nX+mt(7 z;Pu8)6H2SI(_~KAb&- z+Z2l1ST61y7k73^T;#Vc5ZC4Ra2I!ODDI)<;vVbb&eOQQt&R@Fb+$Uy#XTt$_r!8> z&vkJZOz;~^tt;@4vnHUSoNOCPFZ|328AAY95SKgQmjs2wpcjXPBmAN);m+DOI^oy( z!hNe<>k0S6ugMbbOnQeCep|?-3zsP1ekb7GkbrxZDBv+C;E|AkhnFaz?r>9~E!-sF z3Ov#3hNGvUK{BS6f@@R3+&E76B2NoPo@yhz4Yjkhk=x&k+!T)7Xd@>S;*<~YBDaMj zx7x_1DIe-ZKG<@VCUJzv)lOWcmN_gl@_aAyyl~{BZRAq@&hjFko{oGrm&s{1vQwLa z*)kV+k43wq8MSn?(rb7-618|H$7M-Q+jk~v>SE7> zbp6D?=)D4%(%jSlXP@BfLsZ{PICEVz6DG!+fuMZ>!Tn9+kD%S!I z9VAP=1x_8$tOXg&l;3w021 z&_xHAk!5!9By3-faVL*)M1XOz)y|Ce7)KIghA;1&76LbOSsV|dxY#%#ak1Tj#Fuvi z(roVvobAhFiRu@bOGFl3;>loOTgDR=nZ1DY(Dj}qAaDS34+;yU} z(4O7tNw}j7W8ohVxhxBR194dv{sQ8%%>4<(Ds}op`8xfse4Tz%z7D@CUzcALb^9L> zOYZhxAeP+iCm@!*^pDEdd9@#XIhA1Pnuhl)CWps4bF zMfvv>b$r*McNpQlz%FkAv3_0N1Y-TVyaB}eRlm-BxGvLiu|uyZs(KZO^{aXXh;J8F zy$mFm)1Owp_$lR!pJYCI>FWu{_b?D^kbek>HOM~*#2Um80Es)?3#2W?UBFo_^F2fh zdYFmsCz^hw%H0{3+o_Kbt>lW0ZB_Wdyonzt9{&-&lj#tqy}@lNpC>Z);M`-{%_(`e z;29!?UoLowV+G$XSVW}s2*N&O?LS^1&l$+S#WxX)ZxG_4>n})D8+~_pSV+N69}Aso zyj7e&#?~31YW<2RiQCY8YYPqQcR}^!Jq3O-+dhmfA)=mrNsONSF2q&Ko@R?T-_4<} zisGe!WXG@z5LbY>GY~si+`;ixIaKLTL6OM;spA;nEXRq7mf<*gqWxc9Dd$_c)fBPR zV{l)Y2qi_!xB5+GcOsTvH0Sr&(1DHxnX61Z28gRnyekk_nZnLWEsRoXVWd)XI{|Tp z>M%n2I_#)?9fm7kNc0;3B>MFS68$z-YGEU#7B*CB zZUZ3Judt4ycx@n&w-%7dTN6l{h}D3^OZozdm-GQ*FNszG;<(YuKpZz(35esG6@k<@ zmIKZni>*M^4DVQvXr(V@tU?SqBp1DV0<)4%3SvdwYy~m0_$r5=3^Y$|Hc-sVWZ#VyDWt z1F;LohbgL>tEh0Oq7DZ+G)GbOfr>h`Dk{uS6i-)VYJqgtw-0d9#@?8XJ2tZ~kyBZg zA(x3J6OG!nG5Cisx+F?`3Xz*8O{cPxB2PRCYN9%gKctCw4HL2JiAdccjRfVr%D{Z0 zfzxm;VU2#$a~g%0^v8I&!k6l~3k`6kw>J6xm`4{_M5#8pJ#4CG60xlXmp->RXh{8} zwyDFKnrGJfAGMJ0oSRN<#5oynY~V=w)X{@?@GklBcV)WR52EYM)FIxT!4DWdm;>3F% zaOi%A?sMo~hwgFcZinu2=uSoE4j}cOg}`AiG3IunrP>W?D*s(uop>hNAmeenl`bn-4;SN$%LHeuPZC$k z1V_53Hn5oWA{e-o%nLxO{Byv;+3NF|^3YAa;z@Y93}d|X={g)*3P^@SU4Yn%g-%K> zbX01ggHm%Z4{Cg8fJnju3 z@wnH4#N!qNX;pX?ID7c_8c~CuV4^RHRyzFqgo4Wr|NcV^q2b@}M93Kay-l>T!@pmP zu}g+&zgb#n`1di<$_)ShTZ~&G_iv&FhJWu+_`e$dz0aIpfBye%__r)vRBPrk!0BQ< z&c2S8-$;EqzoDWI8vwEYbXXsV{inlvKndO68jA9(JJd%}TmvLN+6zd0v?q{E zWLI{4D=9K70;y*%2b?u&U4dvpzf3mLChkF;iD|q%KWB2iiGs(}im!6vEH=L#g|8~~ z<;49CoVq)GiN$nSljw;T8L5NU4{V7P`4^qq5HL#{F~FgI4sEH(YzCyc-vl@`z}c9> zGY2@^5LJdI!GUZ4B0Y0>vjy>FjtGV^HQnEvXfTCq?^A9i+%F*5W@p3`+l+E(q(eI? zG9!T0Hah@^ZD7ofL`!*wxVoRPkI@HBW#Mks{K({=r@B9lIf=w)Ze^ZG)YS9!Z}8Ky zClE!9n(ER|;W){c3%}$ijM34a2_jp^oB^cO`842QM|*l1S>|j{!h&*)=X;Fjr5Ujo zu*hm>F7_BND#v)a$9M@bPMnAhrkS-f^sTT8e4@`>77*bqaUF=#qg@LmJ=!%u(xY7s zq}I6txR`+5!-LxL{`FNv_5A}BNrZd?CchO7lF%(clF-dSlF&^+n$Y#Y*$Lf9)G`U> z9|D7T{)0f$PCfu6?d1JH(oWv1eC947tpm3MSJc(d8ra-i>wghJ#(4{gN^{ULp$doOwpJMv-$x9o$B20auMk(h1sYqg@=YMNUgaA=Y`y$qAkK3B z6^C9_6h99nW_S)r%@1dtl!G2l>x`6z|MQ2utV?Akm`luRSheDO)* z$To${ZcO|Pu_X;+axZgS?b&r|mwSPNp^gl^Wv&*<>#yUfN0tXpI=p#ji4;959C^GK@siEEhu4u$F)!q!kpmQ_kZK7lv#F+Pp9`qXP9I^=S z)&tPIe}^cd;8zZPsmOc|q-uQ%oK|ZkR_ha@WOjMqay+#&X49?qW!|1hns?TIG8KiS{)>tW~ZT5P$b8*As}p;Fas)(5eot;?T+tt>n;(4z1wO z@(wNM(6Wl+Zb1B9u(&Ibd>O0@5L?nz1F5Y#0*4xx4iuhQw4I4cMLU+iXd{pMp(!=d zQcOuXtzY6a(uCu!2R=5tWw^OyNjf)3$@YVOz7*+dXpfDNshmX2E&Dah^45J# zWi5qHZkmO$hICqZ#fO@z-{=Gh@6ef!b;N_Wr}>#An+9!~nvPS5;(b2PBl3rdU1Ek4^^iDaeXM9s(kS&iaMRBsOkhDiF-Vd#61p3;vTDfU5`=J zWgZa6=yEg=$LMkt5XY!KQu#U@p{UTVC_YS4{vaThlAi;_Qt}4^v6T1#$G4y3o9_4; z9N#p@SFb3l17gY1WFVFtO#))cQLUry>!=fzYW4xretQBiHB}m+xh^dB5i@cF&Ee!`@pe0Q|y(AV*wc^IukvY{Sdd95%UF^r1XfBbd z&tR1@m<}bTwwC%)%I8lV^4e5P3oXyk;@u%?31!(*UJ zHMmbv{x*khb?7FCZgA*2Me#L2Qjf0&l6rgMe%Du;v$QH#6?~O5*K;N@ja)=JPoA9`~+}TF+WALR53546+;ZRUm#kbm`(Hy zQKq3c(^|m|1y{CI(Z52R>7rlEf_a{2`<8vE)5*Qg;hszI@G$ol1(Qp#Gw=r_JfHIm z2oCW%G5e+M4p8yQuiX1Y&nWHob*Z`V1=e=61d0i=V7UxCA~D9gEqUx;4r!+5_FLn@QnCB;z9%?+$i+{ruB5m7YzL$vy~ zS9=!FW(S$=?lRlWWwt92XVxqYq}k~VoZ4Qoa}w!YtBGEV=M3q4-~RYZ$eoB>=Edob zL@)E^bO)lBv({3?kYTN5h^97?_NTqtTdXnBvcy5|=6S`vRIO1xF@9SUWcJaxkD{ms zNIanzka$8*ATeDJAT`~}z!j6}Q4MxU_c9cgN>?HhFF2IW-;G(7s2O?cO|+E8ytP3M z3O6by-{Gw9pBVl#Kp9U-0z%c0Pxfj+}nqmZ-e7 zV+SHmzO18eL-cZ|--i-IhQW6vTDp#TjgE;%5J$>md9zK5WCF-iBzpo$k?a8^MKT^p zi)1(8iYaL<4b zyRUIX&#>cOL`&PztFtEBn>bQ-Bqx-%Dw+^Kcb>^K!j?N|p~yz=6lL3SP{ z5i$FY)D(`J-0jlNX+NT+6nwU*;2|K3f(HYMf(HSyg3%mDJl)(x~iDG~MOIZI93yo2Fv?GG}bsh%2Kk4<%ZIp3B2IGoN_MRT8n!F~o+U>~8MF{n)0#bdr$g zz^=;8Beo{~R1&(j;MZu5POy|V??(~6uAwoV(hbhrV>|OKvdV(_^~C4GyT}&s3kHkAf60ET|u<6PI@JAq)bU4O5d>;uP1UVUP&IB{q>UTlqbU)H#0rlcHK_2 zvevkRI8xSltJu0d39?w?2_Uh=<3QqAk2&h2j{1nBKJ2IuIqHK-H4gx(74HRhN+i0s z`-zfK)K76NRAE`~xul}KqJ5lQr7RYvdCp7dX-&v2Ex3L{y258O)5EwqTA<@*sc-wyrd(4P+d!3dYU3cqVu?pGk5*XMo#;?KwCeg@)see@lWnEqQJG5t3{Z2H32 zKr)8^kD};HAc^q>ki_^Lh+|ZKrhHYOI`oM{A3O9BBWPOnp@tRS2VzYN?*Xxl-JdTik8d*rp-W?qk|w8P!!V^GEHcg}*l_qWhjDw^4Rd%57BsEu#C2 z=5#vF^+DIHmn^@GxXWrdPdvTA)rB9b?tH4C&D zS|u4V@lQeZ;{S!!Gw}~ZsGHLWJEij~_V|q$$Q~uSOS+~1vc{sfot3SFf)amN8%X?N zEg)$~)^yZ09CdX^UCmMZI%*%KR@DHpZ&vkks3#DwljTQau{MX7N$5T_P(P`;>2`JzhYGX)@R{Bpov8|@k- z`J!G-)WCd^R_Hns4XX&$L0Si;&v=+;Nf%X7kIh-F57V+CF|mdEsKOujP(1`n%JH^BOvj)4S}Q$+5kw` z=hg)dm4Woq>k~EA0O=~}CPc%B>2;BsH&ZsFK)->Q!n@*3O{esXz8=;eFu#3A(Jb zI(HP7?TCej3oW!7%d7O>@Jx8NRUVHbJN<(=f#gs{4LOWx*^O?rGttpX)y~vgGS2C# zRkn={uCDD!jK1;p?6S#?36khm+5}M4Oyh|wQ00u-lPD=ic-ijwMiW64oA2WZ+Y2~& zD01&Ivdkn;LTwqwXc~y(;!}ad#isy?i`N5buIhjjy5of5V%D8ZhsDz-Fo>u|AQ3ea zNJPy5Qc(@Sp%aFGvKQ?~RNqMHgl2<55^4jIgj#_lp%x%bXcllWA-mGZhccUql1fBM zLULm+^u%0{W=7XLjVd1Y(apsk<6*=&y=hWnmmbfgm5YC=7ypuSjF)?imz86@(qp`$ z9OKm<<1xf2yQ{9s$}^Hz1w=Gx?yvPkToYh)X1d;EysjMMiyq?(0miwS^vfROO996D zmNEViMCnvN020r6A4ojsJs|d+=v_yB$5G!_s(A}Y{pxk#z>?cy3eH?jz{ao45ndxU zodpM%+!py9)YfWC=w!S~9Bvw1LVktFWtQCDB)aeZ@aQV{a7VF2LACxAtkNuf0;CrB z2sqd*eVi#7b>$0B!slfetN#I!3#$5WATFrtzks-)I{pd7DpmdA(C>;0zXHiz;};8_YNswqS@hWYoNGFk;wsRHViPe zQ|dP;eK|%R6Cs!fdK2AQZ}8HUg{xlhPj!eH8H_bA%u z$`Idl2AldRxi3Z3TP_(L{ZVu~WwmOthM%VUZ|xhQUPLiKYz}`AkBmUd5Ot zQr9M0$xSj_?lh#=A-Z0t)i8{y;_VT+2fh#%$3q#>p~XRrFiMLDxUl{XZRgN74sE5# zYyqTYvng=UzcwdA(7#NyCDF=Scw6G&^CWVtN(upCbnK@eDzb@&5KG!5?H_qVP@U)K zf`VU71j9I!A%#7TXfv6ed5Gq6BLTceMFkSfQD-s4Qf~ zyrewr)i+*&FjeCy0#;iwwIR;pDW9tGLUw-4S&uc zS}Nx`vfu!kt7bIZtb%TUlPkd}&qY9D&sUV1 ze_5&dmy{a62qaRTQ@-d~<%^zCKJyfi+VgSXEQ>u>q?K6gNup&~?0KTi3R~<2q6IAW zG|T(<7W)?blGks5#A06qiN(GG5{rGQ)chAp&3~@c_%k4p^0D$oA1Pn-q4Jq`fYf4d z0tb8bw~3Hpl23@1VUqWVCQj>Bcy`R_gCb2G+72lXq@9rHQ=$jV`7VXG{%do_J>0~U}Q*dy_!j=aQq@o&P!*YTy zMOY%A^@>(tqMMguE-8@=g-@$(8Q#ceFIseHP(yobzmdMJIHNtrf2p7bpPh^TM=^VDqU(Z+Yv2j zL=z1p8m_9)IEFr@hsrjHg0SY|1r1;cp$1L+(5c{%>`noa?A8NGcI$vRyH&Nym)}?U z@)MOWzmMaapwy~8m0GojQVZjOIJK&=it=NCINkiNK%8!V7a(aSM=4*tlOi($NS$y8 zV1GvJOfOqwI}$l%LLD`ZWLnrbT=h(TG_g#vmolmrh9D|1jk$ zrtHRXFeRjc=BTCvD!j2K&jWet7D7_rL4g~#LyDBkVL(zYbAhB>4h0hX9HM;j!HTL5 zP?VnyB(`k>65F-{N!%9Yi)JZb)TDe-qw<-VKx*&lz-8<`gUA{7o|A|=77xg9+nIVK zu-S3>HwqxN*ct85iG&KyS%k;?&q>L>jA;iJFm-bBGq4qxx z2qdnS*6CF0s5tz<>u221mL1~+!Ph-iUk*i1OgI_}|Sr}LZ#0vxYmw=>9Uj&jeeE~?y^f@3djrduIo>UY) z3M6&!5g@614+C*r^AM2E8SV$JX!Y0EQoed0F@#p}ALN+HjeM<*k9$(ZblyGE$B3Be z;U;>*i#$|5mTCv9$%80jPm_P1`0JaSv6bPDRkiAUQM^R*6X76Ru{+%C4#U z-O5b~#m;8sxmrF2ASs_bkT`k{NLyOm`r4;-!JJ77@D1|H5aD!58 zKnVG+MFXcK)lSmPE)b_RZZwHbnX|?20K|KODD`CzhsHZJ&Y|5EncaZ29CroIn$V9W zS}@r}9{l_;M$2&+qKZ4t1rm2W6o}n1I>=GyIO>6pdVr(OcGNbd#`^K$_jXfJ5tL6DT}$y{wg}bhS3PHZ+5%UQ1mv+Bea`#Nsb!(RBtBO;fpTt+kV~ z9#AKq@T%4j zBSmKci7%W9B))J4kodysj(VD-p6aM4JL*Y}dZJRz2|!v$js;%Q)vRNPC%813=ojY^ z+mf$l9ZhUaWv^x(m0&5onsp@6v#w?xLB#BqW%u|S~H#*O{Wc7wX?2^?R0I^GUTpx&YRlSbmTifxi<@nZge5*UY)f``6$Ja;s zDtkLrqo`vqhk7ci?4hV?c}0cgfcWLD!m>be25lK2Id!qL^5r@+A4UhcYURsya(q=l zTp0_MK%DbJ0f=*+%L8%F3z4EqqbOeiByWHH1LMl9@NXd5F8&or8i8Mw8vpF5Hvx&K z+yJD_@wLFk;!BSrt|w~HT}<>N(Tdx$I10nhXGK3yIJIm3d5e!|kuJKCNZ34((eOwd zHmk)K;EM-AOw<7x#43MXJ%2OYEwbh)80`Vog_hCv&3t7uvF7?;5}iD-A?n;r1h0bI zi6h^=jACDIQC_=jSS_TeZH>qaZhh9*apK;t7hN_jHqs{dJF!nGS_FG0tU~TjCIpnr z{YI4i>g)aI%e<=&vZzBBBrGzyj%=W!s(6gXTJU>`HjMMD5WTU!zHV$2Z{d3dr3Y+Gp*t~6_w~^?l=PPl%Fuswcf${(XVmn2 zQyxM)v?XL}OST2cPkEqz-!C2yBD-`v#-W`Z8sX6P4h?pwzarBQNUOuvz@gE}RurB& zIvGb)c6skoGIi+1vqsS0EI5{;wd%?4s@6$!v*X_5UcAA?Tc5bb>8hb z6@F+UAYoWSf?RbDu&=iO29jbF^ zvO|*;nOYz%(|v%$WoOL3L<_oCsd8B%@Yorgoa~eB7Z5oY6^ic_G@5w;zX2qv zz0On&^zw@x^;JiGg{idU@Uo*m&r}L~PWj?znU9tno?*T?ip|kLx@>y{aK-*@O|7Y8 zMu$^4w$__lcb>8nKe|aTnw40to!FM(V~3UUl*l=Xcv2NgydTTBK~;9bwL!OgYJ1+N zh?CBa4c>Kdk%-;aCy6knp&69YOM)hPoXE+JR+bGOBf_jEzTM_69h>M;BH*T;KUk?= zQ95>$`@&LH2WsSUpHZMaiJ%=PLflSS?nPpl(KNYs#?Zu7OI9WKIm@J%lI$6Zpo~Q< z1LYMG@d6QXlas9P<;IQ^WVAs4rV(|)`d>%ehuf!Ag%?d()%nHFw-PvAn9GrI=!#TF zkFi5J#x5RX=K$mKDzU4_xO5rDXg3gLK06jj=CfmfWInsAqweCUqm|lmBoNojjyox; z8sX56iVDLW+Cfpgy`qi-9U7phs=q_qDavoFsH&es+bGI!tthtz5a%ejIS}V4w;2%U zDBc7}2KE~RaoZI201_Ll3M4jI1xReLvZJo#s4FVftN^3~tYv}2$2yi#idV_yh{Wyi8ZDe>_+&YP%Bkc5_Oeyp)J4!dzgGdrqfs;9? z9!$Y_AzMnqWzCL>M3#>3^^iQtbW2o~M?wNSY~>L^?68%G1F^$a%?09WS#_wQ{2_`e z4+fIB2LVakIY1mYe}MAk+Z4sEKvHE|fTYUo3nW!$B9K&>eSoCO?Cq#~IqC$ZnmvKE z%8Ua}->Q^~u?JByD#lczEvaJcM@(&vMUP8)elPa?S;SR4yS8D*payz}&=rS?YKfE? zT`EqK)g)DM_NU-UX@QN(oUBeWDLAcYvab?dC#M-xbzMD#eobyRaZfI~Z!fP8!)>_( ztunT82~`p9;NXv@vi)B&b9FEs4`X(WlN#`;m!e5Y9(ub|LDlk~Pn`ZWJGsYx37vxR^$ma&9ZV-`ptyN;VeoF#$&E+yR+mQ#E%!DXPU2`(jGKU2_kgz9<|h@$I_ zK&)$g0}!uLnX72gKIp_SD4J6EV4(iY>!8 z5h|ncHynCNQQ<`({-#6Wc_99#L*ZG6o>r890*JrCkbfLVzQOP)kYx4|AijzfKMW*` zf_E#ma2HeYUG>79N{tr+u~z0*AT`P@!0D`S#I`ljhfHh2V+}hsVSG7w^lrl&$KVT{ zjddtXv}1fKaRAO*6CV#jcg2cBQ-by$(I&M`nW7K%Ce{eFMK^^zh%aGDRH}O@&|jt? zC#O~Hhf26nonNLvI@b&BT<^Ut`DN}gBGKok^zdeSAS?Y9ge)S?X^D1;ALTW<*N8H+ z0V4#j5wx_|Z7i%7Vw_wwO>Qv}S}^D+iKJuT+)5YNGR>_B#15QW0f@^qx3r@!<)~eh z8W(`LOyj&mIYrg~AaVAV>c4?la`j(8EV;uUKpeNr?~1B_1L7s$>R*8*#xFn;;|Jv{ ze6J|}&Y`arnXiC!ruYSLqS9f-k56lu{}3gk(tV?`yk+ddS21_=jw5)RCgaBw*Zb3F-%6eXbbAhXp}3?LoaJqdnR52Nt0Fd8XWBtWsF zK+zIAg197W;IVx#iJj+*mF5Zwn&{}D*!E$P=i@xl^NXVAlg4!7Vjzh=fe2}<;ZDZ! z6s{UObM}G8r+}gwpF~`t!Z^(rOB!>6COS1J)|Jv_MbQnwbUV0NN4a8;c6p*?4pCFB z(Qs-~X!YR`Pt3vP80UJ7hn8b(_ZSZ=$Jp)iVh+wBMz{NEOB8v+xGPDK%85EAd0UV{ ztwa8d>53E*IY+$ln)7A6q~TC>GdKzzNer$GerWo5$F-9Al%$I5WWbs!H77W1Llv z@feTs0Ag&x#v3&wwuvxiULbn+m27o5){8#B9OH=|p9_R$2J7Bu18MVgCNLkt(Jy(pZcrWrUO@Cz zE7F^jp>7B#C@uzr48kw+#9ja#JV9|`rabiASAr;$fh&M`ux2g;Qe`jAj4a9&?o|PW zdsIMt8xSiP-3laDxdlk9ax)NHCAz^;uXohzlxnU8QZ=pu#v(yuBOdR@8lAVkC+*KQ zL=W1`L<@;F)9xGDnfcH&Bmrp^SH$=^xg$pIJ< zXpsty{R5z??)MQVZl~7H^wgGls)-&XqQ5MD4Q3}OmnRS|^?87Gw>T+f6rhyd&F*fgnSAT8g?;<%2NdFG1iNPmHioh|=| zqWE_pDgNJpr1*bw)NdX7Mv?g%NL}J9;4GK;FVPZJatu|;=tl|G72=Q>1nfwmjadiJb|UH|`m7&a-`3R=K2kP{NZKOWHDWeKG=eDhx`#bO z%CArir*KpVd9T%JFqnd~+a2q_dl6S79_LIgOViOvI}o7lO&$hJCf<_!ExJ6}xwCh? zajDKrMb7zC5KGH^fepz9jdJxKk7e5a57!M*woT(jcn|J?- zp*ciD%U!*>b9rhR^1#z#8lE3qR1)f=ok>C4$K3RI&d_Q)PbWM^+-oOLaF9( zAkE)g;Bvm&PV|%#vX)??`9z!5)P!X=*JJHM3fjpA)I1F=qUNbUqUI?;qUOntdXl4_ zs8n+Tkg9nsuuIBXX#}~yIi4s%1&eB!vpi8}1R`8*&1%l`7#9#@Xt?Csh;ss>Zc$Me zd7>@|Fy5tJbeYF^Nr3SQWxUE`yduE(qB36ZFb*I5(r)!1)qN{Zz`G=%sdbusz}m|A2Asxh8IbXv$~eH2F+0Hcn`MmVf+$t! zP#~#7hX8REG6w;vz2^X@bsoMlH5f8`j*b8Wck$+MFWOLdO8f zRQ_lnRpzM7$fArn!DBv_n9CIO@tML*bRx%fBO;8H79h`9{-*Gfp%vPsYd{c#UJWD$ zy$VR$qzfGNd`CUcQO|bN1&(@_Qq7q_n*Y;)%Vp^dqNmy|DJ&CROtjP>;9Snsg+!UY zzyGoH<(Y5}Cs%5PizvLr3feSEQREkai7k?U6^Jd8e+7swl6x6QW^OM4$;|Cphn{li zaYgYXKr)+q7)WMw4**FT_X9~9_W^Ml@x4GYo4ZF*e3zo=b|5LS+km9R76M5XxEV-i zb2k9HL62*vucuHQyVxo!>;KmgrHtd0KoOtN#m zw?WsM@CNY)t^XELeCuMKCv^bJ37>%|4)ZCHILs$N;xHcpNmKSAkQV=Yz%QXev|F<%g& zj6Pox-Pgxi#zfz6+_Zn7Vfz&XsV2VwNj3QyNUF(CK&sdGz@hmI6|XVh5w(nRKN3Ad zx&Lz9v~srp$z81q#z3NA1SAUngQ%+DpCrcCE%kL*o&WHNUP;T;>~|t2O-`O^%x^>} zldHdpo{_6cDCy7b;T)-rk*BKo$x7&$@TyDY+` z?_CB+`rd9pDzq!b^RtrhW<0#I0T`sYT~ASNZ6Fq!TMLK}Q03MH;)U(}8jf#uAojA{ zYCv3wxxPSBFKd8AwO&B7KH3vVmd95B(v_Iyfx}9gXl0_MI>3Xd6r&Z1QqrH9XjKYN zE_zCYm55NQi@dbrtJ7K>ET#f6x!%N5QdlO}hr%(jt#6)PUzd(a>tpt46g3a)gk|K{ zEhEF^*C#4FD^2Qonde>__sJSJ|BIGz1h}}gDu)BH?JIWxV%t}455%^w9N?(^9d$dU z=KBG$lz4L>DWlDR#CoG=>Z6x!r6jJ^4$ZwKtPp*EW9O=iWb%} zT5cN-cl#A%S{v}dKK8<9CvKzWwkEccLKpL!%7IKpu_o8nDtDA{yDz}TeU!%KmD>`* zE?CTS2dLcV-solqH|5b7FiI!0E0CIJG;nadybH-qRXg`BmpwcQyAvZVZg{PioctPJ zPQ>1xh&_o>7Zu&qPQLN8S2-d3dO{|aVT`7OD7{GokTllQfTXdW0;C3+3|uW$o+z}FmWTp2Aq1;KJigG6ciE<|ZiE_sRsdDpy!%3UyIHI}jOHXlI@9Ew)DR?~b zB^87YXCqW}SO6qCoCPF0oDQTqoCchw!x=>T7do6td}$r>mmpMH!;67Ln~Q)%n+t(N zoAZElba^gtTAP3DI}_&a!yNbuU=Lrtt*Iz+Cc4{+dqZJ-{8f+ zmU!&q6@40L;al%`VEaGrr^p}n0)iIC0$ zwPO4q#s&Adn67u4!w;q2ON5e6h`&@Ky}!N1gK_;cV3piG4J5gH3P^JIB#_pZ$ADE& z8ZM)w<#DQcf(RLzdzNUSJfNz+1e)}KF94|#pD!cr6{2~jazo`8i4aPL*W6!c@#6c7 zJQ0apEG9xmF5U=<$A=ofL^xOP+!sJxy>p)farKTrRBHTzQseiPYTgD?)4m0qmG(PC zE6tjT-z5TSyIUUT!!#HWeME#(E74f#(_kF9(tpBnP)a|5BBk^lke1T7#GhfUpTe?C z^gR(uvm!eqSUb~J(>PS&&_9UFYD9kliO2p4Bp&+*ka+CxKx)=sfGZBDZP~T-<@TS6 z)Y3MYK88g%k=z#>{zn9Byo7)JMudzU|4p=D#$?@uR!kf!#GO}1wm1u&R|DcKbnXkp zS?JV9Gu^Sb@^!3HzK*?=ud1i=bzfEay04;q-B(t=t}7{DmlYLtTEU^^6;&;#D8HJZtEDoUQ&mGYA2l6C`K`Pm7SUV!Y3 z3y^wgp=;-Eox68lrt`9He~_+6G8Hvx2$Ge-z+fO53=9I2!N33@b=+-%gB`wGrsz)` zDQ~R8Bv!_51Bsd(2$yWQQD7Q25WFXOI5|C(93nbd? z1ElV|H*i`T*!iCJ#*4vkbjb?0jV! z1*d#PcYsWzCwo6)47GhT0}{mBOw>fQblZo*JOJTh$W}$B8Az9C_NRC$tyE?U(b6(? z)YC@9K=;NK)Z`Bgld;sX02I;aEFdY`Gl95h3nwWxccM~rComNs2`?P&s7E>KkxDgl zfz&k*2F||ka41nTa(x=n5{LBo+-!6zg_o{SU<%Y8ju9WmF;atgJ#B|Gb%=UwM%4LB zimGBz#OH&T4QkE>(%NtiNyy0Jg+%kpPREeZMMNmcqlwPr@N`bNsG-0eph}6}?$9j` z-3-LNQ+yK;_fGMRKr&Fc0!UT96gW5uYB=y%Gc>x(9N&-17yqDq=6fJ*N526^4NDAHT$lEp5F7kHP(CkhqCbd+%H-xnd~qOc^P)9D z!8|y=R@<@&NR_An4lc(U5YqMEec!s^NywLBjH^MEey9@=_d}+G7r81kGIfS1#x9PT zi?OTw)0m|wZr0{cHzFRuuK?Kb?{&QUlkW`zmu*#zL%kGL_5fnPs9Y6@{i1ReAoh#= z%E}k72qgJh0f_HFnB{=ftjhw2tw;GXxs{0SbrV{d6}n9ISjp z%E&UqJqbIMVXWL8L~-WbfW(={0*N#40>moCI|GR`j{@SoOS2P@nt23p+L@Fo zp6$$Ih@9cfrfOHBmW+$U&GFUSAQkPx1Q1le@x+udW7v~usbS1LTq-8oD@%C4>x#m+ z2CkSW2YQ_ky=KeR4D>krXK@;{9g}DqWOySkEv2?8wq@!d`<8`EWPg6P1bH3iB{IY% zvT?40hogdExBP!@cO+J2tP%xZI5LICsk8G-d>oBscCADOV46WNJxn z_)9T#=Xn<|k)5*=Va0dzILBmh8ndgAFd}fKdr!+}yeDG!ED@a8ULNP3S)9h~A*5hp z6Fm`om*Jb_@$FlNuioREEPQ;f+-0-7ddN(Va|UsyJbo&*y)b@!Ac#`w+km9fw*txJ zzgekfe;^%+HUgucZRHV%6f?iEYs@U71e21fEdE6kODoh>N2}=kQ9#@(<&Om7Rw;i3 z5VuNkyYiVsfV76s0j_A8Mb|w20Z~1&Wz4}uOlffpXIN4<`NN5$>;w13u_ruqzL)Yb z0pa?bkvYy|oJWikC$>(j^`9OR4UY{7(ITl@03zF?>MS6(N7b1?Y>&bjKvH0*Gs1J4 z=0qU1%n88dGVfs~z8!j_(S`ce&%cO!?wV6q$>Gblv9yV4fvy-&{Lu+K@r?;D)>X3$Jc}A)|hj7}s`VB3hDHgtm zSc0ve#W{lAFXK0^g(E@3;?gTfl8u*t8HkOSe+h_<7rzK3e)@u<=s6&c5j_jUF`{RH zIEHx|NSl}^fGhC2>(qL`^6(tQ(1(tQI+O!7LA zdeI`_C3?|fVoG_@0bD~(^a;~S_7vWwV0z_yj6D@XYX|PLHlAM;y&!-vr0sgWW@Tw0 z$;whd;tw5w#7H?saSS9~P6WhVPX2GGBYyK2P^Stk+{J$aF*W)HNX+#!kfiZHATif} zfwUd`0hrA-wrLRE!15}boDKSk7*eKEH>!j>a&c?t5ye;g@G0ObBGBFpuZH*!a7EQb z!r?QR(Y{3NRa0D6WpYDfTl$gBPDE%adF+f0A275n5Kv(9Wf7N+WtIWbtak$rw%^?; zsZ^F_Pad7zN}hz}iBVTt+(2$Q3axLPUCg2F{G|n3F(6$GigCuKAWE@r;?R1G=*{VM zH7vi5Lu)IF*K%l0ht_bYFC*#(`v7Te=>;qw?%Hh@P8{(8IJe{3lZe_%*gVLqhP{cB zvXDB(W<;xNYOF7Pdu}z32DhM3w0f_yMH_HRay4EzLRSD9KHBw=xfs(c~R)7 zNg~>Y9#PMa$US?Oy0J(0kEwVp>k|u|+H{LW1F3xwPm_Q3w0p6^_ofz31Pw2$!P$2Pi`4y5KvMTd0!iH;4x|-e7;rin*)JMO zl>O@G3`!ge8)8ETCPH_h(3DErmhM8dNq8_f`F!Y36g;_h^0fM0o0|N!ChEB=qa8u+ z&*a_fn`yyhK>v;U_usxA&+8%z_3m^%bHFfeum%j+Wzf*^!-o$XxcfkZ0N2N#|+cI%sMdUur1{_(ZugfK<8DD7L+~ zq|PbAYtLUjUpT&zQ4_NUw61q=fx8qS?mAse@%)d(KcS~ z5sUqhi+hzvPQ<&C2($2ZX?dak)6@Y2#ta-k za^!IQkJ{JK1BVQxxldONDkox5N#8>jRX4iEi;#8G->xu`RV%*G#;Scv6Qeu+4B+NpN1z{l7~j zG2%!u;wogsRbT==mJ#P}uzns#4plhx4OJkfe&w1E+iGo9Gi(w6s{|F;oTHK!U#YECC0sW}~iG`-!s6hGwU{JjGh z(n?&xN|4SNyL1DC$nOdy@|Ok@`AY$*{P7|AUBHl*FPRkiW*JXH_e^G3WI2y<*>a34 zc#O*j7}wKp|E%OOu2_z76_0V{0OKii%AHEBhsU^T8OEp&h+@^=Kw{MzAhBvMAhqi0 zO9e`;Cm7OJJ&KjErM4y*ME)8;B7b!tk-r*{%0DwCzb_cl@+FhD)YkSStd+?Oi>&K0 zu2YV2eUEXya*P{#j2o0=+{iI<*=*weG-l&Kye;`Q1ytV5W8Ab1W3)AhV&tuW#K>C$ ziIKMeQX@ayHBg9~gCT8XQNkLzpC@6POlD}joyWLsImQ7VWB&l-)AZgmIr1Qn@wnyb zevw@nuyamX&l*@x#1K!!-~i)Vn$|FnacF>Xs50*0G48TLmQvf75fP09QHpFQAStpD zKvHBo0%?)0x?-TnhJztpWaqJZw#at&B#a6$kF%dBHoJI?Yph%`tCcU(S?lt=!;CH$ ze~cG@*K&-zd5mMrF^=;Xch6)r(RiX+eeF6C=?kaX)041AK*ATA`Mo^G2?55uw$S@{ zj63$oD)GI`iP+Z@F)_e+zos?GWBjpKR$8^?MAUgACI=Wl<2ypIRJ~*5+BVhwY0Q*B zytDMJkOq%&S{cTu5k&FxnLy&_Gl0a;_XAQtUw8(Jv!Uc1>U6NAJ$*i_VLg48C!;CA z>~wDS822y37##qj=sX)pbZ!F@om+ua=Rd+aw}2(Ba}95wiq6p?V2~6J29gvG0+JNw z0BH(;g;O{XEa?>1_fjx(JsF2)GGkbEB#5HQ5kR8K;XtBFJCLgKIDL||w15r+OInqC z?S(*IJUQBvaa4f$Y3=Wh@fha?7{{vj9qTd9FT)s}1fpc_L?FrB2|$v$0C!NOgia9K7(uVC98(kIf=RTjl_^zuPjbNFn!{UcXOAQ-t816{o0Z| zFnwpHG-UW;PsT@zPMA1}k0UksvH<~1PM6j}r%=MbJ(s(B7bmyn(Y#%gQ86weYR zXbKa(Otd;$S9dGmd!u|z^a2s^EwGVvmNZ3sk!W~9VP?zNrV;dEW85mXvs4qkM8xEG z3q|59MEF0feF=P?Me+ahyl<0~rj*c9Zln}MC{RNwSGk%dX&XqBkfbe0F{Ei)0%;O* z0QKmvD&hqq2&f3ip(uDG;t7f;cmN`Y_l*cDryvOG|NGt9*=L^PjaB~7CvTqFnc0~= zW@q==-De+u2e~=*!9)`2OZADCYq6L_`YpKS@%cA^v~B;AGN{>1U7g1C#=j<|Fq0x2 zBA*3|l;z(5Nm)JtNYwllkg9nMa0N^F3rU4~;?m%nZ_uYT0Py>>hy}jLapwmBFnof6z>8^^S>iSRLK8q(k9FQ&Sc2s zKY1t)USzJ(w!$?aT>>1834|hMxTFt} zkam;jIKoB}(pAbBiQ!;y{08v&aXKJBP6y=2>3}3xT*Gjal;?#AZwV3J93s3aM2O=p zfU|jhL)c*l-4}vEbYB2SbiWIb=zb?4Ezx%XPKmhQHd($Y4w$(hDSNXU6Aq1`}QB~SHH64N#%v2G+G@2N;k3@&)8FM?Yd&KCe_ zHQz!R8GrHyrZ>Kol)_Aka9GnFU=eHjIv}y8uK^Nkx*d?Jxeahu*!^}Q{`+7MQQre3 zqV57DqV5EwqP`7S+n%3Ml6@}UBEfw$!VJlT^j(rNd6C&b-^A_%gQ))zAW?rWAW?r0 zAXR@i;7LXO91J4rXMjZ1PXURjp8!%(4*)LA$^9f0=j0)hGC7gCMxT@4gF)2)4v?t- zEg(_<8$hc5mw@vVK~iA9CT%h`e+Gl7c?^)Kc@&VS`4b>j^DyAN8ku5!gtW=jJPig> z^AsRa^CTcq^8_GO^Elv1MZEw95%oME5%qUKBI-FnD(YFlSy3`gD`i|ReOkRqtWX>Q zQUU*@fC{efCDJByeg7`dp3K4q*H>foRRhi@SfX&C8fjAQc-&WE^K&nNGNW2dy|wgRhjbj4R1aeMEya4ME!w)MEwDPRQ)`_Sy3`6 z>_r_01`%~AAQ80?kcc`2kcz4YoNrvxh8K{QNwLfado{;^LDU=#NYoq!NYpF>q-u@? zJgKN9U=UFafJD?{KqBfmKq~53z*$i#i&zc@5w#4Eh*}CrL^T6aQB8m+71ag?5w!x4 zh-w8SqD}y$qK*eVsiwZ0wE#K%(YM zK%(XhK&oZ{@T8)~!62f>0EwtkKqBfaKq4v(nBNd0ygo#DU5KzLM2O>a0cUmN!zGwL z^riJyFo^E+0EzCm021AA2Bg_|6X29644AE`@_WD_W6+Bfir;Mnyo&-V^cwFZZL(hD zLNa9PBU#Z1y~brm-=!6!U=H94!}x(o7|Yj#DC6nt0LcUJhfL(FD@Miv;H0HTw(szR zS9DxAfhq&m8;!gh7$?(lUFbTlkC8G-5kTZuz#@tLG9Y;Xz7>$v@aF+Z4c`JtYWQ=2 zw1#g6T&w`wkVs1p`!8auN?#-)oktRAUSeY9yTBodx)YEj>RW&$QQrh4iTVa0Nz@&H zG*Pz$W_j4B@{3)0e4UiS@`y{{AuaP%eu+!(w@FE7ganh``@kV~{3C_pdjV;Ren<%w z?Dz+yO=idUkRg-kWR)w}@k2)6gB7Eo#(xE(7|}0Hl%Fw5#)vL=h4zbz!dwa94Y{g6 ztH|S%KYS1)w7^UP5lj!r1Tj;lG3LENlKqozWA?zG^LMaQz{D( ze*sISls-;crIbEJVsT2#{{l@?`cH-8mjJ0vy}-B?Z0dQ^CbOv*$&j(BWHB+=RQWTB zz7pW#D6pw%Ac{?GW1>_uO2($Ha5goyqA<0o@=hwUyrV+#Y(Sd-nG{hW|J#!`S^jq* zLni-!3Y}cY|L#WLZWW^-|9gWd`JZc|?8zvZ{9onrzgIs2N_1&ohR%YwO&XUiJf1HkZ%1T;j9?($e#F6WG-em*$Z$GHkcSli-_| zo8pZRcG~64tDE=<_`YF&sywJa!l|z>P%rcI=uM9_B1-&5`P$S9VN4W!LO2oEtzI5$ zA{|p8U6za?z0oi(nS`;t8bm4ERe;0`bOLgfS$(2Xt2+Xb&0rO+&HMtvTL8UIz)VBP#98%N`%6{%Pa;0n8MXy>> zDLUVz2@8I3?R{VrH?$Uz6jd)EF~d^CP9L25>+>YN7D9LK$5o40FtzQ0+6Qd<2;EeowmDN+CC1ZY}&NBeu}ho+T1B= zrB9NOP8$g}ZKW@PN2=h9fTRk(07$CfRzO+>p9h>xf^1=-KuoiL9SkDsYk)-5?SMqo zZGcqNR{>{5$tD*OHT5AdxDuQCphDFT1n7Q+YVHA4i%-XvYwiZ*%CqWw4j0Ges=EU2 zPCzn-{Ekw~-&SgT2Ov$_bzHw^&)GYQ{RpA+JkhnmYf1Da?fpWVBV0qm+Wrj#_6#GE z+zTiW&(UTJ_kZZ%7@xa{OW!1AV5kotvcU^2h9p%KSAEwpSVDDN{R1K)%XRMTW$>EC z3MQla`=q442+h~<*?UZNKYz*->gpHZ6vy{-K$`iV04{v<_h%HGaXwSs5lxWw>mch_ zM%GU&$@*o1tndxT-x>+OnT+xGhVgd=jNu)QhYjN&3K(^hTKRDhrEUHdkhIOmOyoxk zBFp}$xb$bz)NT8Y{|l^s*D}hb&#LGckP}`0C!)9-sD25MQ(XNbAff(UspaPsivI>k z2Q*Is&JSpwAvRM@n#5;GD;m)Jn}pne=8H6_(!WU4p3vuu6tPdW(ig~;c0ewA)$^62 zX9hIo?Lq;~&;p8U0jXh516(O%(@9IuTzn}h`Z=IVt?miP6>e!KK;o!&1SF1XHXw0S zI{;F>vjA7py9a6MuJy~V&~_%FZ(?*5ZQAa%lk!8e-;E5pC02>NtEUx4+=WE$?lT@C zF758HqkVK&zK$r%)sA@PgTNsfJ`j*(_y9nX;dy{G!>v;KWAVg?R+NI#({Uh5bsY;x8sSlZG<}P*kPXm!|>B~X^Jgj2yG8D0ZOGJFak$?$6cX@*w=u4oA-lbEptd83QE zP>%y(kitG4kkn&8AeFu@D*(kQkD+mCJ!zSG{AN;*uLD^`ZvrIscorbZ)(9ZU)-WK+ z)({|7yb*9JTglhpw!CV7K3g$zAq+qTFq>8Ts zoXWdV{87@5;K;NdJpdFM$&C$PFuqFoF#m%q7+!dT_8$z-3dr6;oE>T zec#GPMuo`s04xNMma&8%B$n_qkVW)Q0f{C22#^%ay?~@(?g1nPb2lJW{6oO0yc@+2 zk(RcEyPYN6N5W*5@BkTdmLQSu_q0Mw_%Vq&OL)*>QYHPFdr_2Te%J)Z@KLJx;k< zMVA3dJ-&!2lC6ILl5D*INV4@jAXWT#z^RlQ#WCb%>hV5Tk1sLOWcB!OGUVz}BLBT!2KQjfa;Qt3Nq1)w-(jVmtgN?N8K?@#J+KFA{aAV5-&uLLC7+6RziYi~f3t+{|y z@m_#aDL0A_AT8YT#aKrrINAnR+}59FpNh zfF#350+I|L0Z21^DBz0qcsPlfdbF>*m6w4*3VSIasmDe@Dt$>-0E$!AmEuwpX_m60Z6jd3P=^7063L$qj)80>3aN`tH%x!CacFTGUVz} zB6oUPVLhHmVy+&$9X3^ulx6C%9~_e5b$}$peSjpxYXND7djMCg$6gXM^(e3Z`Sy4g z7^JXA07*S=1f`(UP0Iyo{8|CNDqW7|c|~wwS!Unn*i&30wLQlTm#YDO)pnS^g9_#hO0} zNE*sbfV3EHq$FEI(4p&kA15u-P=3z5x<0Y$c931&m2U$iO1=unDJg$msqrm4x(QmP6@Q5_0WAXkQ==sV{tw@QWm-M-~$6%OvDC;FHK~V`W|b zyLII9_rWb0{~jPI-R}a@()|vlWilmeaB=BQ(lVv{n^3ytUw|y4e-6lrF8vsgw95Mc zNvr%3AZeBN0#e2I0IsC?r=+Dz^LMT^?Xr?AM~_BH}n9BIXCnZhfTR5 z%5o*_Osw=MI3&Y=0wfuJ1dwF-kAO78zXx2=4LwYvtp-fS$!e9Da{Q!`bf0VDWsPs8|90b@8@;opYwUj>XZScB^Lb4lGsfK%0n;t7WV zlnrC4fKj>|NUSo9Qzm1qF^ts(jNvu-Z4Beo$r!gYjMK@Oo)7q|Gq4$12{6)7x3i6e znRy9MI0-uxNC`lgWKAv)^cx6@s@)7F) zKqF!Qyo6_*gaZmB1jAThB)mE=;aMl)-~tK3VvaBp4$Vt&qpJAu0tsqLr3MhiEiDEl zZs|Bc;+BpDq;Babz!lxnF(hW(k}R=#w{*Obw2aK@@_x?cwWXpI)L4g+up%$vc_*R0 zK!O^1`7{uvAL;=lefw(w$%^PnfHbcw0avi9)g;-R2CIroCx>WVsnr_)Ui&E|r+ce^ zFx90~Nl34xz2pdMNJ#hXjNvQG^BC+Ids3 zTXRdvp~Alge<17pdJtpo#*7&jW~&duOfa6b;UC&n6Gv4 z$2g4d;_SMOO9#VTIIb;~q8i+JX?k6p9A6ww3w)A>d0Ub=d_SwUAkMZSj+Dh-)KkE> zo$^)l0Oul}DB50m=2g#RS#Em`=Syxp;d~xjb??$xj@GI%WA~Xu(Fj&bgpd=ia#I>mn_Se2E@om(Y`@ z8C%>{%Jh|9&%XZtstWo-nfSIj0-a1(HXrj4pI5DyZ}Y*l9N^{G17@1Ve$x5L(z0c9 z5aLnReB7Ca@k+8xdj?Cc9Fc`mH3Yg{8Gn9Ak6Ywggeud zZfDjewy8_EPeb!%Nxh7pms5(4RlSCn^2nQ(TZ$Gd?_5da(Fik*$N2kk{@#FlURN9? zc9xZGXud5qdRa|gRz37s+Y7QhKkbTXSxSD7OWT~J+2RuZq)u5*T1TW)r(EUecn>pg zTHiu5WnRHP)ZSIr3YGPW^32lga#Y&#uYa#-IH{+ryR)TjSx;+o+p_NEw9zFsvoRA> zV*GZ*@pipv(a{q_LpXSS-9b2$*jf4Ng(=ix&$)Op`d_|5i6_!UYep8X=^I|xkAI{6 zYvgRlu{Ah=f`>fLKb`t!RPWdUJtyMVgE+NwC4F(UqIpHnij`g6JxiKSgF#?e(ptK5zf*BkP_1 ztRF5#Cxf3C#~35H`*DdI_XL|s^@nk^M&oQ*UoOwwZgy2P3m6^yY>%4X0pZPxKxq;; z8#o-X-`YR40bi_2;X3=r`bW?1Ul);YNAR_bHE$fjv0B^BE=9XYJO}S8a5Ii^dWy6? z5gx`d*1hAyqo4X(wPR|{}UxHD5YL!J^`3(`;xHPqEVZtSZR zv0j&r;`<;jeMz`&*rocZZ4r+i6NhE@G!75plbYjW*7 z>k-!98vygia;i_IwJLfe@VxYilX3EVarA$I&)4A=@cGdM<8U;_%hm<+%vyT0(kkV7 zRqyD4JDrjCk34JG=lMLu*F3)!u*)av?tH*5|2xJr+02=ax8cXM)cdsXp482h=I-#Gi7o<-SM9z|Ogt4W`;RD$LT7k;tM_dFD3QTyrL3vd2FQcXdbN>1Rcw z!13A~2-4Gz0gsVM8-|{)N19+<-)pi5I~(Cfd<`65uW~&a!%h7dU=D8r|E%a7glS0) zeSQ5S<73PFd)J}=rEH;11+5JoyHXTC9r%{9l|ut(O}HMNqXf!ATDtntT{3(w!sx+# zcviF-Vf;lM`1(bEDj&0=Eg+x>IP%yyFdm_Iila9H#~!N) zVlPLJKb3Jq6=eLl%jVa%)H6* z>w>#La7%`V2a(@W^d1S*5?lGvE#fW(4qpoC!-qngDyh#nx;)L>*^lL^F;_?B`2C>i zXp190lzIFC;CSVFSKA|@m)kwsw{(T){UGkUCzeE4quG>RZb|MrWE^=JTeE&Gzjo(u z?$HDvj$E_ecbA}h2u~<{KDiQluELLM>1Fx>xYoC%sKsg$8oMxp?cca|a8s)PP0*4? z$h(H}T<;q@SUux)z`K^`=TojnSiLdhT+1+e*$?B+D^D}Mn(t*l0$%ofH!!a6&4Mi8 zw^vz67xin;UJn}iZp1Iuaq?+hspan0Ils2rjI=Tt8yeFuRJ`TLLOpfzDiE!FNaey&!Jw1cNeZ7M{ zXRjY;@7B|YFmYAw^qdQi<%BYoa?q3$4-+*($#Y zT2ZU~7Vz3C+3N2?xT${xOGoWPZd~MATsi(e@Y&W%eOuF7e*ivS|EaasqP70e^WKen z(ppVsY->%>|9@I*e`qYBwcdkt+>0N_6}5#UajlJ}*49)LLl~)|CESPj+V>FFmUTa1 z+p-=&SX%@6e#|gh)`Ph7%F|3s@h$5i@Umt7gmHb#3bKH=EsJ=y(4PZN_%b?Sp_Z0d z=q27lX}{WU{1mV)6I$lafD4w%g7Q}N3*fU>Mc%Eks$YVS*Q8d38moa-{mS$H8u!Gi zD%9B0m+AA@4GzM5+^p+wpzpW%ZHsiOPnfm^aN#qMmKuhV&G-;!%9=dRTht#QQ!R?H zTGSr_TZ?)GVYLGCJQA`y%F~R6c#C=zytJst7}r}=kOjQ8DC)PqntXp{Jj>^O zZ$h)r)1^A%)heF^oaoA=8&+v)iB%T)%|Am@wtQ)se*r#NrWU-j&YI|P(DFW0@agB7 zn&=7e=ATK*@l(KOEu1>H#=@TlAFoL*9F{Z<7XFOq{TuFyg=>ba(=?WJ{L8fT^;mI$ zp{lh%3*FD*=dAs($;R!Nf1DUe*|)=Z>wX?G)Vc|)b^imfweA-YR*NLx3k<`$U&5VN zo@T7oTlYV~OY8m@<9h23vVgbNP5suoUD|M{FLr>WW_pNM3oe76$jhV+7HnyW1>aE6 z^Zy%C)qECk@IqZ07ivva&X1>vkEGVaz`Viov!O=^j#@u{%HYR_AP zdt!-`)cDJ_!VNSg*dSUhpCqS3?>6|k@%5yBwsm0bXzytH(b40)CDu~Td8lE`Are+g z+!nC4#2E;y^^k8m!?48daOah$8O!pPSO;EO;`WT|EiuR<9`(Eh%H0|Z-UWQTCbeK#3!fNw^}M^`o>;IZ+WJysEic!K2gMlGm+lVT zd*J7+xNuK|mNI6?#<9{nmU5?xgdNK&!=9AoEt#-d@?LiIQ@hqRW3 zxF@Y7WX3iVMT#}P+?F!t4IWK}{(d3!ABx|0NO$d|qmFLOS*KbF5ut?~4jI})2&+XM z3D{cHB81hV$ae(8Xdy@8&MQwd7Vlfg(conZIfik43kkA-w=IPFZ3`jaV#c$4zJ(+- z+d@ip#H)oj0#0;g(hUo@w8X-1FYxupLXujzD_d{f#{nO#+YvZYsg4?eFY@zCfKRNu z95*4HwP4EJ8Vha)AFoL*7*;|*ztrm%3+ zVw;o0_z}tpkfByfSgm#iU~9E)2&<)$ua#j~Z9DF~@-$sc(vNqfD>Jrbi-;bEwS4B3aqvhlGJLYY-JqX1$?ktAvmjD z34D>&5}$qwtd3RzXD#_8gtL}RJzHbRCxegIf3jry;@5cIQ*ci#c@oQcxjuM;MwyJc z*Ff*7_;E##J?bX($gcLy&N=R;ss>&yhde!qQ(G!Wr{T^kPcv3UD+Et3czIvTxZVnb zEZ`F>lsy#OWtZoRqkh$i^uA8h>n)P5kykkyFnseoj(WPz92i-BI^IU?!Y3YOeKk@%TRV^3<689@ zp4Tt%ax@nBlK9vk8RYPY_L)*!)LGG1~U08NX(^lR7 zX@^_V-rgFya9QGXb~cpwi(r@Cn?~FopVbR} zV6ox&pX%srzx^tyqw}HnZTR7RD(CeWE-W8k{^C8QmwG37S+C^RaT9wvhA$uuW3_kT z&P#g@)|>6!65dku?gC!crFvt_`zVKw9X<3up&H9xvUcSC}WrxI*~~WNeGPIT?6Af@y!hPwNN# z{e%8~CGJ{(u3jPQYT!aSLrOrq(X~~%N=D0kQp6^YFo$SgAVK~}kx73@3$8DZd z5C0aywV+dSp7%TnyB$W{LrkmG20=aV5V~t143ocQ%=clw*ASc&5oDq_;0rj;OADPU zxp$;0x*Rf0`vOlKy$?9{*w8K879pE^w66jlJx$0LJvVn&(UYq;CXVT*=XME2T;-$Z zmeh&};#t$&nZ*;)2smnO}mPr>qoK08a;TE`YeL1=j+_LZ2zW{Koc&wX% z-<%=PGZF9holf+*w>Rj`m=bROruSp`lL+Ijg|Tsm2X`xSzqyY5I5&xG3z`RZ-9K)>7qXF70nn4oouD|Lg(ei z;BHfDAIiITV5ol-ugtMK!)VT}BHjr-ci{(Fw(b(M8)!YQ%Ayy!GCXdfcQrm)SY-n8G+_Stz@l(AF!LMlP>XPTL zx)i<(79}N6+7fLMNqOD1ht*)_Uvl}Ykjt*|-tn;{k0F@O;jeuTk6{isYw6Pxd+_?` z%GPdImdK*FxHz)3Honf<)!f+L)+CkE)6syOc-~TqxBUdgmoxYtR|xP4sI4hJAHNOx zSi(K-a9<##?W%b1tfC3)#KHN`rS`fF)~!E|&Xn+;9!tXqnjPHt;h zp@b&2zH;OY4(}IB9ZiX7UrCu%MQE1Yu)l_G=OJq6C-w1%kPlXXe>`?zI_(epw6`n=o^wcmIi@^mEpLZvhnzQx$mjwh)8-i>aVNdtIBk-Y;is$Bns@)E+#3)6iG z^fPU8m7;%w#!H{M$y=27e+NFXC)c+c`8-9%#uKS{H^-f>0@{w%GjS%9dvv0D)-&E= zD2S7PBa!KKc8QV;&-cErSeHfhQKNcIJ*tc_e=xUb&)f0CYhAJ8&Zj=KlWCA8V+k4M z*ZOqRN7%GYSDij>g>40Sn4x^8{8<0C{G=_EqB`aEGxe+=?r^VX8JfHyh^rjURDMHK zS)ncpZOYB2qAQi9OHQpK5Nv8TUIR|?AD}zC*bnwDMdSLdQgtEy==T)E5zGP zTVP|paT zsCW9x(X$E8jRGj|IpFZ5MZSRhyYTUKeV3)~0eK^j{GR5w!C;-k5hXwWKs>&6i1bt5 z0Ny3%qgMY=-$1{ty3C5m!}Ut=G;JE{-I!rQxztAg0#EC3Z#X|-3Tn2i_iSHowNVwq z%|m=MsiSxN^yYK1s^VS*l)RUsYS8`p1WgkgH*V7TEWXv}({HMD4CPgdrc#Hb+v}Mb zZ3Em6NNdw@QBs-bT^$cT|A?nEJPvQfBMld)yN6x5Nj=YuwuKzdPB(e<_GpI0%}H& z&LO4>Uyua6W2x(6p|3)&YCzv7WZ!lo=vTq|HITNL$xW?Tke`D93Yw zPe1L?iuVSNSMKQ;Z*YbuMZDv@08h&Mc-~jyzK^taJ|9z&cws!>>sZc;pJ(taYsDWJ@FmCZ;x4FyN6MZOx767r>&$IhTxr6thm7l94OuF; zo^svOFy1`}?7Xb+5f>oL9+mudR7^Ze0vG3<}&5BXi-v} z4q9x8C{);5EqbkEA#W}uJjq^*y%FG09)Lr-eVJ;H)_`~kU#3jq@GULr(MJ9H{o#mh zl7kk69RMyLXeARHX^~jP?d~1j(C_BHl4MYd4sq!b zk78WBC(rcymLE>5u{5gWN(l9Nd?Q|y=wCh{C#hudD+ch6DAJzAcVPus@bTp7);GUh z?;1zPu)N*LIj&1|b8Red=iyk6t$e|@v1294E4tkL96c1h-3pXS^SS6O#|F-JBQ{s2 z_{bpQ?*lr|*1^^RL~LK5<>Tms7AB@BD=x<&%$1dnezycGBq+t-F9!c+ zYb-8L<#-A3X>W+{00Wovgzz{!!}~RXrq_Y76$mGo^*CGq4*Ti-gE($U-)>Iab=JZ+RY%iGE08w5cChVzt9(=S zj%|p~(mNbNiqGkw)s4Ex2?xQg(f4;Cjx9lkPsA@(g5K8!8gzux^?rUa_2&5Es2j3) z=?Qcz5vJelq7KwDWoj+22CQjW3?9->A|K$O4?6btG~C=}X?#LR(rNcYUG1%)6B4B? z#K$ltn+4(4FIFK8E5QPU&`(Afeye$~cVr9>3x1|HdX3=PFxzj#IDiX>aSZjj)2ldd z&y;d*-^rs6kr|(Y__?-+k;gVssk8<(y{7t)86M-TC$3uy87i-j^884Hu$4WwbDu^x9`YDEdn)V?2SoDN>bA7ET>b%7r#3iA@6aXyQ*b>u~yl(it7#4|LX zR+k6J7((2HFOybSoTVi__^sHB!14GQh%G);%Tp}tOyKdXvU+&*Ogw-!4d~f%gTTY$ zQh551dZd=tgx;yIpG#&#u6uU9I%zoVrf20?zwe|&i8XpyGpL%miCZS;D; zEo0b}GQj~F!4Wk~k0mf|hI`%H%4MjHI@ z`MC2cq6OdV{qNhsOaJ>0#?8tCKGYs)w2q($wwm2NZD~iCG-TripUyNu=7l~D@Ahd> z{&N`4=wbguJwh|-E1Y}fBg!%^X1WF!LVm<_3FK(2cyA@S$T$0)LY<&-t%NkaY*0f3kLIgr2;dQgo%!p?S%bE%NEo z3f>|bXCDgnBOafalQJ-A15NAM*)7r+%Fd@dMqKg~rTco&c(E<89-UVyv;nL)oH)^s zy@DKiX~28kQr!(2x9}YQ#=!w@YG+IVL%fe^Jl_`8j&1@z$#14D!sA(5GQRj7&d=q! zQ}l;=e#kok``&_mUi2=6eP76VvFHMXTk!hH0DK}&f^{R_i-41ngp5nxgRq~2lxg2h z(i#~5V$k4SwHtIgE)`sup7ZREcA3yj7fl_t(dC74!XEN8&-z|05o7aXAWJ&EY&YKF z3^Kx+-@HGX-t&32#H&dE zPzAc%UgXnsjqu1LtH)hya4Y=LR9eq&-g|gNs?zgcr~D~APBsI6<;XfLTswB^nBWW! zPxfQ$8N)M+M&XmpN5IdqXSR|_`zUA}D{3>5@m?I=2t3DizCDC7n;Xk%yjUX zjdQ~8DaVaX<7hE7&H;{d$dXo4W%m>3%s8VGQtxM4I1=x`tcV&Y)5m>ugTR?e3~6}- zc?Z{}?2~bLhVOZx>_xKr^+;|Xk(w_n!$C3-dOfSbLG3Vs~f5yPh`*9=y{LKc>vQqwA48GpL zw;GuBp!{Dn@cJMpuA2?Kj#0lt9>yc!FgQ zb1_n`JC2q!C^ zwb7jj>#8B?cOjgt8dgW&L%8xPU3K(*()|h@SDRRqoI_0#BCYd>;7hm8S@GS#@ydjoVOkCOZR=zBr=+3v{RDSjw)N>=3AUz$ zw-o(cd422q8N%fG1!d9dEI(k|irB&^OD{`f3nT48$^mR?OiLviTVEnG-};hx`PN68 zD}#d8XKA+e5wET9kAN*-C`aG=EG=n$I~KIQUqVu-Vf2%1A-@7{bAohZr(cr~txsFo zZxBveSxxj?p|ju5EZtLPZS*_PZ7cje!bvNvivECb+Zs7SyzByt{#eJ=RhK|4B2V<*n z7F~E|^rty96&ajsou#_RA;-2hhM&NXt!)PGytK6>G-LJtY4}O-vPC_`xW07-e&9nZ z*;jT?`rV$(IvYBc_cX5T>}+o9?pf8`+11kCMo(SanS5pCZ;+Wu{Z@Q!B@^R+Df=P# zG*=7BD$%n@%X9e6XSu+;F^}rqo(HUDNiRX#3;1z1UIA&PM8wUqakm>u~2V%`E-X$wdTQQ*)`W zV3Dxye<2=A*zIb_S&}J9l>QvZ*3yq>q4c*%t(;}>WyK#`w9ep8EAAh_TgW|_FXpQ$ zNZHC9*0+`_mE}**KMOVI$^`go{M&5nR8Ktv!)qtj zJEIUV)B`r&slB;(OLwFU)IIH%R=E?yTi%Tp%u|CC)Y91ax`E-uPwoo&yWw{Pl(1A-I=t8q=;)2DG+ks@$Xa*^u{HXgtgnX-{`SI;Wl;}(^L_cT z*LbyunMdMfL>)U=z2O4j=WSs*$S&nW5Y}>KoP!xgAHNWHUU{17v;7GAQ1G&kKa6p` zXAZJ}4}Cn+#WO*y{v&-H&5P5Ix>{uPr)lOS1V6?&%IiPc$&zuzQG8Fgr?Gv-3JxF| zmp62x+?OU}hhxDT%6KbA4S|>SvGq?*{5k@DBmAxf*T0>oIu5!Q zl}7h5PB-$}(y#>Yr{O)GmSt_MFZLv+u9m+{-7Q}CzSQlT4 zqyL1LslV0he--unujReo>BpN@o!vbR?(BiUgNXEWZM5{X)heB(M7bqg(1W(V=YVP7?~1znUZ(DKUN`$B_1)2&RKAlhf*KoN|1$M% z@OlrXUOygS->G8)J06fe6enMS&a2ixm2gFrKaInCK3-LHrp7aG&@f)Tla-7+RL{l= z{CwKU%P&X6%J0W1Zajc-!&$&5dI7NP=o{Q2ou@*bA$0uado?8LL38m!YxM-z( z9#@&Mn~Fyh3}FHgvkR<8!^;;b!hvK>{e6Qtj?D0#p)ysNWy$K*^UA!ALBw^tUp1~@ zIHmp^P2PcVX&cJMa|h~3x<|4!9;SnIaL$Cg#N4hqJi6&yCB(Cpadnd4Bx}&LAdRGP z^vqRCt~`RoXqdQ^GbxnLw^R(>oy)*EnNg%AJ(F3E$AHVt)8OfZXEKa~rk6YQ#y29) z(1a$%(J)4q6X4~DmLD(X(+ATjPfyqh2Aq5P@Lq^dp2YD-uabT?uZlK7j(vjK#y>%g zfS*rT*zo>T@rl4l6-uM;%%!7zqV?%aFib1F%=ysy#*jDj1ZkeykdM?;+nh`oZ*8UM zdoYZAbHBah+r#G}FaC*$wDXuZd&-Z03bSh!2_N=b_;luFn5QcLE%>*Bmrr4D^Y^z? zS5`OZ_G#>5liqiMU(@dJOgr;r@RsL!KK=UqT?8Jc`#sc|(ZTrY;V((?*%N3=b(F1q zmwG-|UjE4)V#99j_kzy6UWPmKn3iku2wGZ}q4}rB_jy_G_p&HA)PdFQ>x#Pat&7hq#}6W<3i9bvO)|SGF zKi4CiJ@Mxwz-3STVI8nX(lTa`LL6uIS!UC%-st&_b{_QanX5I5JmU=@J9 z%SW9q>Pt@iQ6r(})&qZ1--$&}^D)FrPVkA{ZaJOXx(PJrL3~U7IKpb(+*iW=B%Is& zgx8&?WxPX-lc&H-kMOA?S-=Nxik8T72zsJYRD%}Hi+a*DvyuyM!E~!9Rld(rAH0v< zZ^4yf%2(R9D!*_(k>CAhiMozZ1Ly-~wcu9j+yWhbcOrL%Qa;oA1<-6hzKAfd;(1Gc zt}W=%K+yAL)#H1A>e&%_;E%mOf7JPvl+LQ?tEv+;^4j!Vf}Y2vr|=6JcweEPUzgm( z-EtveIV?v5DEqG!q>pLjuO3QxXZpULN#7lszQV7^`7ekB?I!H?epB@p#&2JH2J#T1 zC&5sz-`4p1#xwa&Jk!Y@gk`1a*ZX(C%RGDw_oN)BM&Cu4SM4j}nbo+H-`N=M3bS`% zT*^q|R!4VfT$CYq2{>B<-}h5|X17IQK3fNd@LEO4XA%tg{2}By*?zT#>EdWf^TxE@ zozgiax<}Ji*e=}p=|N=(`Fi=0>YE$S#6ptO!@5^}EDyrmrA*!TrF7RsKUUqvJ8-*# z;zkFCd_JIhO*^sq)Dhx?DScJZL#nSZul(vk(5Qi+^QWrQl&!TN=I>`IeN&>Jt3J@^ z7ZaN>?@k+^vKPvVa+DWQaFwE8sT{AHyQ9ZYruwsS#NSPcexq@gaz!rt-Zb;+mb(`% zBG`-*x7-I7R))~xz=9&cQ&{b)qLW>ckxW*azM0}U78lEfWVd$D9r(6vQhg3aRF;;E zsIKYAwHFefy=zW?}6jU4Wm4g z{2OYrFN1IQ;>nm>Gu#|P2Q1U>Xy)9&9}qt?B8?vgE;o9Emw5$7Z+~?9s58QT5UO-- z#SJw=2u*==1CN0BIp}NvFMC|uTRaLHpEYx5dV$Y9Uv5Dh^ISfk&)MFaB+n$~)ZD^e z2K&+b&s{z|$nV#4Y-z$wBXm8Ebhr^1-gS*~CtEoFB&Lt^B;w3t`uu3~DTFm`l=lR~ z7;Qd{JFgrKxG;M3X|VF1Es}>3;d53VJW0?6TJR)8@@Dxn;5E5*_Ww8FHd~~h7yCQ# z)&@ATXP#(RQUdO&E`A=koL!*YtFZU<1;@`gd}^>t3d+60`@$om!+kjRVa-O?8kS6H zL;rx>7x9}5x%qN5WsGuUn%#&2FzZN1*6e{8{x^O_Z}|lp_(HwF_U0f=y*Py@?8-`b zO_`r9WzHjgOnF5y!cyMiPZ(|;?mJWc4#O*lut(-hX`dy8S>}y+8+M?tcd!d1LC%D- zot8n%l|5{h@Age`@feqBl@x?hr_G`qtH8r@oBUfx zry(3h&Ya`$?U5tt>7>EtYjNk5r&UE^~QO`Ey&Cs|v zmo&X>ym}yu+QH0+qm`OyCxnyHN>#Ko!jtXi-Gy|spO>@h z+|R4;SJ8_RcOY=9aja)!@2G$8Tks2kZ$8&;D05MCKI3TMrBlX!}GoZ_l|;^U<=Mw0bBFdf3<9bJO6f_J{Aw+ zv3LkQcCsgHPw1Y5AF`L~ZPCI~v)njY&&_gk2HkmB(=(&-rFg-T3d4Axsx5r~8>fV0 z&R!Nq8*$-*8UY+6n@#D<8opYgANoZ!KAKs35^>zmW zwjS;v%JQDb#-9&7uOeEQi^;_^9*keiK>T@-!}|gL{%U`(_xFQw&*=bv@M!SLOqY6t zg@8jkkt>_dEX}4fOACAHxT1i$uS0!$4)gcJaku-FV+;^nI^l7w{>0;)k6n&@Pe<0e z5SLf2I+CfXVDo1p!svERyt6B+MwpFbH zu2QR74VqrlOZik3wkowej7m>J{Lo6^+nDRD<=1B{zbzAXqjOl`1L$bf0$&R`)&hGF z=E#*i*3ud7B@H9j({SgdmXgrST$>-ct_3eGu8(oOWe0xXgJprn(J5(q1sc-BbZUAy z&quxul$nu%GBC9CsFI`6M1FoWnxrc~;Lv>8A4jEj8qw-Z?Lq) zitqNL(bTF1 zSsR^!a55UjPAG&cpU&#(6;EcZiUwVLHw%R4H=KinFu!|;PX&$)47!~wEAXt#uFK8M zT!>n%#|X0^LxIi_Xi)(2CpCJ7;idXYtdgX{b2H%?nJ{+FrucD&O)89?uBk9~HoJcN6G+IkE$Orz;An9j zS2NT*Z9$r?cX|`T^iFQY!}P-pzlAh-r#IuyOTA4(GpionJDmq!dZ)KCu5ZnOANb&b zymul^FH56mNodBiAg#=UdaimzCf_@$54p4}m-K20Ev;MQxjZ6X_Ia4`loq^`FDLat z7eS}xOO+E0!O{{Bw4lHPoexQ7F4=jBw*j}=AnA$T4qW1iYNK~BoBD7Um+lSk0fvJB1d1pADa^;s_~Rfkr$?k}mpcMI^mLRq@tQxlUb)_7N{r>d2I-≪|&Xn(V;CVS3ysAxC;#H6L z-+{Xh+>^8?dYi*PITa>G z09UkI?5n_McZ+#kNVmjIc8f6$cDLAV2xoVTeGRzmZZXzDgx*BUm}5-h>~1lJw|2MK z*TI|XZQ!N3TkH;}i~5q?Vuc3tpS(22i{F5(u!Ah?X|XdQe1X&SCVOYbdi!L;_?Fey z%)=bEF!UoB2f7=`gQp;2RY>VzQ5n&@Avxqk8scF0Ke^xm@f5z_X7^;M6PT) zvoxE|EG;_^^MFs!gZ};y?sgsq>(ya))9Oz=UD533eUOrx0b!lH71na}W6;v`FeGwT zv>g2eG~3GAntlpgb{^(uz~x$rpWXa9XnM8gQ{mf21)H8TvkZ^UTdSwH4-2&)x841Ly$|A??!5&8bW zFrJVe!JSv0W-QDm&jOd?%=X+s>fbmFe^cUdK?<0L&tFMjz8p3WQS^;$HyXo#>C_Gw6AEaw2g&IT3vh4})=g30?%wdK^9pz64y(yBszJQ*`^5 z{t231J7h|gt|e8RT&dgoCq4g`{20@>o-gm{`pLhb^WXTf;T3;9U}@{H`=sV70`i3E zTULynJaUzygJIFU>|P?gH4Bdu*KsGUdKucZc%M=oVe4_K5oYUfJ8wCn_j(wfN*cUR z4em})j%IeudhfFhcaP0cqsX@>-Iq)o8`^r7 zG|PuCn`jsE{ZLD5!gpjlTb8Ws4&N8WcPIH()RkTM5@k^4fLW(T-OZhQRpMq#mB z`IMBDPkLU8b6ZKdnz=0#&($p~Bs(v~cuEWNQi#Ws4-=ZHNAouA1k%KG+4PC5a5dG{;eIQk?jCp)_M zx^hw#nXvzs9Zph8cvf^Zw&KfCjXb^At+W{a)dSxovafJ>^gq#yc&X7tTq#-9UxXFs+Z^q<|2Ps+>3 ze9-HYwijsb`x)K75Zp7zHql(*?f3P_yEkx^R-s6LCFri?mre{0x@lt3@hZimH2p5k z{&oJteT=t1=#qwwBitjU<#m9@!?yF@K_9`TFOCib4SA($x%bO1bgln?UjgG>FIT{{ zVweVwirC`0!nM2?A0>0!z4+BJ*&)3Sc(rZfICp+JaJq`7D`gxZ%}09Eqo!H$tAXQ{ z8#&n(G0c^}3Nsn?p7&tfM{G;x5GrZI(i`HLYytS#o6&*V(F?-vd5ONkd3SiDL&0}3 z(w^H*ZJxXEZOqhj*V;@N=W?a!IAbFf?q?W14BubU9_DE1wIiit5avk9z2js?N(>)I z8b(UT;?9d6mR>%gnfH?XNNF*6$v%F@fy*xM&G)7vAh%!58l(N@T} zl=_g%>`GQbOY7EnE{_Ex2F6pG9Wi9%9l^@RrDdqSNGsA~Wu(dwy@;hHy~upb$8ryr z?M0SAl6?b%9=#Dbta7a##i#Mwus={*Z*9~BeA1KDM$L?CR=SoUob&`$(Q@Ih(9;BZ zVNb)AuolFhtmkm_NqUasU3~31;^+i~<=qyqGHGT1Upx-qn6hi`>T&2rx(mFBdYCna zCO*-v);n*-xb#Z0c$UTw2E44`Q6AsU8wVbaxzBAa=wIg#{dCW}@d}hd)@!u^$Ll|N zt)1}f*Y0^ca6dO?IdogbXyCX0i|0OB&mR<<^MiPJo9Z1;gq}|P*bsCKmORgyR%bjL z&Y`aKM%TpnaL4%Q(L5CKz@~op)rO{iesgFv%*P7L;-*z;x?9_m8H!a%kDr-xvnKA` z3h#|bTa6z*+ex_dQcso8OuFgWz;iNqd4CP#W@P~%ycay~2*T=dPX)j8`oL4Z;(Lhq z9{06?NjnX{)N>*9Fzq2-R*z3tFXAv=YZ;g6O7WX_9=6=%Pog~qoDGwev+M86(Hih= zo|I8$M*6@9uYPWaHs1H+m+bmM z(`(A)srGZG|G_#c>1ypc3m&r>|(@jW^ z4$5hQQ(rkv?dxyIO#iUqX~qQzO^bk zTlg#XrRRV?*{+duN&hc)jj%6UjF!Z=J^e>qLVwf*JbIjojW`Zk=Nj2hwE}NGQ-XY! zTpgH{yO$QnGA~TB0&S{%`k=jpK!A-_fw2*$nyr zhaU_rHM-s@GwMAo6JC@Fqs4^XCi`K&a)114Gb<*1C&&40SoZnQc`?c^H&Ta%9*DS( zuka1>?$KTxz${aq9%i5GTk1O?SFYkY&l1xK9WNro(w9w(YNC}ZS9VArcOl|C-S9f( zQLl3m@UGSPHN^KItUAee0mJY#7vs(=PcyC8dzwqYOHcFOB3Zx(Z{p{A*az+HW%C~N zqMR>Ry`GnIT$Om=ue>M+=1aYl%De)&RDbF7tn%o2s9X5r=*l2Zc6iu1rX-DVbamh> zxv#j`^1OyRQHLtql_yI7+7L%%^X>Q|zUu>D$(=K8Wq+iC>>ENHk^N5CW0CBSDW9o_ zi3;^_lkzT*@y&-2E`#?vfB&$*f7IV^^!JbBUZgv$HleOggKx6=`c#EDp9Y>6?fk$@ zoX=K>bF+!li*m4fZmAIG^Cr$cs23aO3l-vgQR5gponTr~_iBGq_oe8|%InMWlL)gc zKjZJ8^Y>f*{Y(D-72GNBC8WvPU#e}QEm>O9yKztCHt<02h7_()U5a~4JKoOf@7*X* zgtMZ{faA3}5ZJrz7QGL62*%i~ciobS_3Qb?0OCIYJkKa`b3>j$Jmp|M0`rO!W2d`$ ztlgpy0*4BK&mSBVeY-_h0f!`EWv<`9H7k7B^Y7-4FxBLCw`_;^U_Sv~{l+ZAUqx8I zE6bkgHiXkXQ#ocjdV1U`^*v{=cWZ6wk6^hTeO7!s_;?ldV8M%`2ipZb*w;Mo*Kwbc zlpuRBEj8)E=H4IKo4t$GQXZzJVWyUA2X8yL0Bv*B7-V zLD2`#{w>5;&;D(|(i33D8nCtE?;^}rM1J*>v_OXMA`Kq@PTYCX;?gwZ!M(@-9(c*~ zea6l510Ov8@w#%=+1$|7)85v4awMx&t2?wXV>Tc*wiUX@rD5({k=|DMQLCS{~CoJq)C$FFNh!kpw^>Yt&+>4*9ALNl9 zJ(O2wB}8f8VR_j6Bi`Pbc%g?YpoJdI(28WH_1T_F>e$siWc&o_Nz$CDV>EC}OFT!X z9~aZF==&T$fTW~2Wwqjmz`1X7c_Xbun5CxWFRi{D-3MIzNdJ(A8Lx407~er~a~jc) z5pEfC9PlzV(ftUgJx@*a0B~uqQxj26;&E!C2N6!ZO-=L=!ilF5I%lZatoUh4&bM$Z z!hLSbdxzFxp4*Lie~Nfn&q7^W<5_+NK3zLFMRS$geFt zr7N{Z>WDB6gC`xcE$(nMKsR1}gy}pFvS6J|Kf`~b?1DFb6Iyyt9P#S){tVc9y1!6X zPIesq6?k4nv@kZw#WSAF?}>X9a+vPN{QYr%f5P9N#6712{J|rl-k5%Eq0ayg`9!X4 zI>Sd>3^c@B74sdKKse zZ`y!CM$@Ec-?CxK>{~VmAe?>6Wo!bzGZVP!kSin%jP)Xir%tW3_P9)8ts|P z9v9Lrag(=fm}L%f4m9I^f!tmNCac#PR7v&tztp9(`-hWNrd)&eLNr zH{Y^pcDksq_$`~Q+RF`ZzZ7z^Z`dT(Yu>Oyi{;K_=ly8q&`S?^0cSb!teo9rZ)w>* z_Li32V{d6;k3DodPr5}sM;p@Y(g8f*B~X}q?G<)sCX08eBdonj7vj!ya>VO&Bdjf& ze4Pwqu4EZhy5N7$DL|MM9f^6VJ?*bavvD{zG`lWGyemVbe=T%czEEz=H$0)G zC7$pV1?zGrLsB@=)Oo_1=rzD?4oF#-I|aC8U9L7-gK)AAR};M!;bfhzD(VsbitBQx zfj-%~TrcTnU5-{zj~3iDJTcnWvwJP4jpR-nkuIrKqJ+H1{<*8#^X*UB^NZ2VSu zzvo?#`|_luIHJ=M!Xr3l9{<~QwmuvqYu4E|K>q3Y!T1WS*I2u^Z0=Q(FUOanI{@=y z|ITvK*7z1!jKZ{d2N$lHu{>Ma-5cBjN}ix=Z^#|?Wx)1qNBXqngjS9=LLV<}y`1kL z4_jlJw-k+fnaaz(#MD*9JFdLQ-=9sMIFF|JV_7&Dr|0LN2pj#5M)QG|G)nS|ncq#& zrC#fGfL&{cUc&RXu#Clny#ZmCH~G~g(EBjFnKXE_H{#Aqy-`9l9>IIB{{t`MZ(&?t z$AKUC;Gz7w2+NjZDlM;sw-mj_=;e8kq`jVUtsH-}F9*!xvZQsrXxEm||Oa7`a(hJKTNXLJe{h8RX|M+F zHt~+=hc!9(egULhh#!oi!rMYP1o@B!&Jj2W+G%05w2MNz&B|M$H}N~aofHpK4qDg6 zkfE*X62RKJh_|hX;Y;x=I)ygSkfzWoj5X7e^{1^Pi8Cd7ALR0~wD(dMa6umOJdSV@ z&(P2cR>8-(R^au15b-Bq@p`9hw&o=Ml;}#;>szpXgs7}TiV{aLcUar7PYd7;5~k+?h}xw z9_^EWnUzo)G;vN=x^1K4L)x87bI2pG2+yU9~cNK74%fS;;Q}1}MTZz8TqP*KflE$_=LY6TIV=CKHSzX?_z!iX~}t1c%RwWt@v4&PNtP7 z+iC&Rm3o-or@|`bh(Al)-@_{M&5-#y{BWj~^IUUqcb*HD@Oj{aCBPfpitw~8#|!mQ52*oIFM=j%A?KhJJ2tUq#+<6w1u5$XCW({diQBZ;MYG%Aft1v|vedhq-Tp3U>Dg@|vZen3G&?q7Yy2T_+1c&8fy=cnKf8So zXnIYrD@X+!T5g#LhY_pwVcc;qWZ2p53(Y8=rJ${c{i?PYdV%EW!djv~pK^=T_=AXl zFoce7}S@$%`coYMpTG-j2IY7cy>-#Nx`uQ!8$r?_=0Mm-xH*vgE2^&l)xF8tITGTzH@Rrf@^avd_AK*# zo$Q|F$B`cEO?cMj6TtDxd5_Fo%Jz8jdD8Pfh5JcKiPL{*Nu#I!ALdf}u^HRUr92J! z&)~;KoNG~$X*Fo6yTjv5hXry^#)Qj>JU{Ea1fL}MBtSZ0zN`H_WZN)LebLb~_tlc0 zvdF_Q+vJNbPCT=O&=gp)(s`%Me`e@s0n7O0CH>zSI`NhsV?%cBzJ;UWWh+~n_$;=v zhY(7bo~R6d^+ZzuJ8xx1x$Z2`CFPmhq8DMm1q;G3^I+`lDMx!DG{uD%gmASBAL7E* zkcXXS;9uy{gvMB^0Zy;F$fYEmrqo#&IGDqI(Pz!rx--|DgHJIHpEGvE{Q5QinRd+> z-c#ydw`Rk{z`Aw)>-8**snD?vekdw87Gu7gxA0FYwFtW(%9VA*a-B{b!u2+c@~8I9}H-2$SFHI?%*ZUGp`b*R>mYpo?WiUG+W=ZB}(H&^TUK9m3?d zx)z#vs_QU~=XLEt9_VsS61tA?arQR47HJ%>Yfps9Z*?7Q;;F75yUHt`eX!-%($>}8(AL4Axjp1~3wzOlL@yR{QTb3<%`;OycdK+h9T44pUH@Q$oYx`>1n_MoI(bBf6 zp%opNlTm@Ev3+G*H)k_ym+)krs6_9+8l{!WoFln=lJlQ>qKS>{pN7Vh z_ZClr-jnfbUyfSaG7aO+qz_$%nOT0RiZjaPs0Cb8utA=4z9&+Q0AQRX*YFIXK!DB6J8AvTnFM}XCFS0b+$W99lI0o z(XesXPp<@RXY7x~krM;26RJvdCz`ARJ!>zji?rP_ilJWScuu6L=@=at#)r8!N!sV4 zCiaQkqW@F1@JY2%UFj6?>$P9HjuQLLx15B=wu7h2HIT!SK7%EV_QCShewt;eFw0M2 zmWRU3zrxIq!hH&Fxx}3rr7cavv zQ#rUpydSAf3xYat8Khy>HL8W@!1+nl>k*z(spPY%{$D%?YG(s3o*DdT>@~4{!|a5={LR&XafkP+mNjPh3NxmE!sFAb z1lk?H)aC2~zIvE+6iM4BjTCkC|4+`MwJG8wt<6Yl<^B1A&UK-3bgrW&NJF)@Y5aJN zR$c!j_A6#09%lDvajQQP`y~3mn4Z#3@HmEv{iE1N(tq(>Bc9L~I$!UpUs$a?#<~6Y zMl{vMj9#lePVNu~(N#ZHFEgG6m1~_llVRII^*!ebbd{*gf35N)xkLPv2GM-ntCj0K z&5iM*Zk`Gjb*7RR@uz3F>pM$lk|kZKeBKabsoxj<+!+0&Fl!aprwr|S)O#kUR=HVn z=H8Xs*!Dj9WbNxVa=0AzR&8dr0>+_p9*pfMSpVkU-aeK(VD;blfc155>KW+l2kq+V zj^F!$Z`!07>bJFHFk~Zn0baoRPl)?H>=Z4O{PC8_yyt^}`x>{+$v$4HQHn ztrcG{s_JX+8VK?hw&F|j1~+W%>g|o%UDAp_%1!LDd|A)td8D>H=DurFP9qQ0L&C;9 zAOF(i?x0DzqbBw4G|AZTYFFC5eYqp@gjq=4QTuHk?LzymDj7eGC%;T3iSr6Ck)P{) zZs|S9b#>~cxRs0IkF$yDXUhF@(U|#s8UOW*b2u4oadwM%nE8AKyz>7_`3-SQ{NQ$W zQ!}5U#h;~Vzl3OaN>0I_W<@i9W>%E&G=Gr!Yn}98_#I;Fp8Y1u64kHakKt)#d@>DG6yuR>CGPqbz*7gt^lc5McNEv>u; zESn`=PDWGdYD1gjO&|m z$NM(57NQx|y}Gx(YqXk7o2V+uA)lp3>al%(>~h?|Xtknw?-o5T^-Xy74z$6pAjqQ*X&lavL(5c|5*CZBO_Y+FYvx{PBMLXUkviaW39W~UltAP{w4nF z7xUiI(u!-a?q7jdditvThWttV;K_oMCdDO&c^WkFeM9~%A2RZ^yv2?*7guYQkuP~J zt;qNe?^!$wX~Mc%T4r5eMIS|LP}aJB4M|DESQ1&+uY+Bife!m7ShQdV^Y#9>aAy{C zQRUmXTU*2h)$c+tT8O*B^xKn#*dGh=y}}usV;QE!(QaYRbnHm!IQ~Uxs(%~<| z|KEoH{~iASj{i+zz;A1s;vZ|mA9+vdBvm$^MVgIgkyhMK@uv`cQ5KP7jt-^4JT)dskx%Go;#wR;%oX zJEhp@LbOHDt}P>yYr11#G4uxehZ{4RzEzK{R$T&CR#FzRuWf&5@mDPk)fbf%WHARA z>m7iMCzE==kFD0GTy{AUTIvI#%bqz68G9!8L88%8-z&r;+5<~F2tNImhL-x?@XD^V z$!{o`i61=WN)wIsv?wc-2>JUD@h!I8>S34C!fu_oyEOi{@$m8%R+gDG>D=>HanxD$ zqWNF7lEz?RE=zt@diB_FobJ%X(Iy<}B@4&MD*a}`f2&B9YPwsmT@@{!yUCYn%{@2o zSNk{W(*rvgd5n|@vE%3{KP<;e?xPjHl=^x$BB1S(aNcu!Y+F{7MU}$GOpkkO}L#Ye@ zaA?$WhH2}WES?^{!XM`+$?ZFbxOC9f$3Ty|Sskz-yBM!5AQvz1 zmvhl)rzbW~Yztu&Jt?hf&;Gy|!|nR5PmK5oG`L=+eods`mw0G5Z*N9t6{uPfeo`KH z=fsPq8hciOhVCY(cYF0k9*wEV5SIs9veI0Aq^-?pin?3Uigd+UWA0q}@UeFO6}Nbr zuq{6^-a0Wj=XV1IfPy0W{qXxZ=&@u`cB{+5Y(GHQabQiBoN?8C={2u_7JoNpH9_W^ zw7OhnR7KBn@4xfdY9r&kAZu-q zB`oRPh8=nh3acp(;OhX_Pdy5!CzXTXr?Z8h4aSecjOo7s+{VG?y;|E+U6WRo*;ceO zf=r)BMyDZ;qT6&uzD+TX$XEZ-#w%<2x@)>qYe8i-+`a|Y;QjpRp{X5f*66i!VrF{x zxmf=NmDAw~rwx=dg%(A8#Zw0qOMB|TnYfFm4x9}Zzlc|Oa>r8#&gDH?uXyS}J6Jq* zU>)zpQwIW;zTQ{7?5P8C+fxUk+nzeG9v*w@fOv|h4xA5WPaWvO9pf5L9k>vz?9_pa zz-iHQrw#-x#oPPMP90Dj?5P7AaTiY==msmEI-opID;agHmO|J#G3w zUJ}NGpy?N_Ms`n;xz9{i1D}i6)L&t5j_}0G-6v zYP;g@EN{)>v4v6`#5W3E<2t)rzer1CvCu+Xi68M?CBGrgNfvl&MMIoL+acZpO?i~j z%zaS!b2I5J{N>`u3`aUgS8>52XEOIbE_s>EiNx}P%zQlaGK>CldJJ)KIwnW5*5OI4 zsc9W;46UeFFRMJxyT(8&-EvRh#)5%#jYMn`ELp&F{Mcp6^SKHpe^=w~*)`$51eAN) z)04ZAxsxTF0mI5~W{ExV1%Ekg7QU{=#>wHKF&F2Z&foN?HjSzOt?-1=O4TD8`|Pp| zJ~6%@U$hM3Q|`%1x=4gEh)Vr|0do-!xOnyDVTxeDm0SAq(8RO4i#Rn|)D`+-a8d0V zXz_QX%Z<<#*V;yI|KGeb*ZDp;-EXa~??(1xc<9PF%i~Jt9PekAjnw_9!t}r`;QQg! z^v5I5)#615Q1&;w8R#bnlU%Hr=&|-ck#}#!{94SDaK{=ezH8;C_CJ~b`jyd&vpUzo zt5W|I`BjbMJ;~xd)qdit@T0|1zKHMX@@M&wp*$8X{*sKaV@6+F9LHnB^SrgnGlIYP zm4?30(wVH{jbTRKXGY$pFinj|hw>M9^C(_Uhxoab_bjaiy-^-5EvxO1p|QaQ&{$5q+%^O;Z-p5?qg)+qv`&Fk{n6ET9c{F6={G+)+Sf6IXaB zj`t#+x4^HqzkkQ%?AVC=Tsh(UfrrRDOeH`y-$wKxWa!R>#^AvCRNv&}j8A9vS;$RR z&|!yGpADv8Q!S6Rgc_%NS)UX5ZsY%f`CL%7x0?4gZo_`A#ode37ItN$aQ`~vX4|rn z_;U&Kc|0_0&KgKRvDT`@#cvc|knmcY(jVLJ2jS82htiw%1x&4rd;5^5?~SBa)dc;| z_!m@OLKylr(@Z`tL zm>G8M7(UL$W7lPMt`UyzT4mQJHYe}!B`JPov}^s)?w)OzgmcX!@1lxuzr$FvmQ;3K zz`;OC)&a;GonY3Ak`nzKSlQLj-A@y$IT1TZd}Cvy4M$*qEU90D8F*FDl%mPkN?B@M z2i3Th`rJzAUhHcdADv*!_!UD9P1w*orp%C_ z_8o~pjbBQfYUt}N$cVosmBAPbudV1(oO1J9d(u_-%FFA^f9Itz(broE?`=GbG2zkX zXDhzuTr^7H3L?fynXZk}%3<6;@-V*!=60ir$ClRrl1`{WD!tmP^GbU8`o1;w_Dk2u z{K$+Iy=mr6ms8XlGfmU(>$U`XnsI+GVa4`R;mQ3zgiyu0=(%v*d= z{2{)X|7zt!;QBSwjP(eobiN;6h5zC3|088Gp{I75TE9S+Y7Y6le+>Tk?(pk8{t1Iy z9;G3s`%}Ct-k;{l^%IaQ|1Pbh@3T2RJAdQDk9?mGd`^#OtvGk~Ip|937x*vT3f8LsYqlKSu8b8usO#ioePvrpF zwj8L{Db+P;Rg2Mz_VFOo=aJEANSEj~U6Jqm;QBcl-31%3tiKdb>AVZ>7WK%DG(HMF z%$qe;G>t?=FV2@KW_G^plemlXWuF0yb2*YDcbqT#BJZ*95$DUk0v6}XzRG)XzARvA zw5532`7*ifd|7nc`LeIWW9Q4nQ=BjR7MPtc`ws3H*EnDHU9htGvb({H^JM``@%DbR z`7*`9&X;`;cX7V#2VlkdGUY*WzD$^%FOz$3=gWQwZ&SYytE~C5AGxp;UN&FWR9Ng( zS#QBA)58218O8artmGQYOs!Fkw`X~zJuIfRS*6xyo^{^Yi3V$!^^xUV&=+@BxO%X* zi*}P%4@>w9;uI~W!j$`$JjIqmcx)+t&AYP-p_cwe{2|Vn|7zv8;QBSwjKvEr#joI1 z`2QXL|E^3X^sJ?*I?EE{^hfv)m?JFGr1-@&>69zQ>Ccjv^IvO|pJ?6whxd%nTB4c@ zC-VI*@Hrh>OB^isU!f}QT7h%J3d--%f^P-#^+J~U570sjkSEeNjPzm)pjg=!;7`aXwg7(t zvn_z!u?6@C@6oEp7T{lCu?6@)-is|jz>+0Zylo30w`~EU+qMAzfycH0;wiQOH8g8m zfQ7hYTw@EcA6Qunun4@^0t76@+xyL00L8(!0E>}PYytKME4Bd2gJKIH%(eh>?`;ck z0AVz>09aVH08638UtBTC7Zz*&gN4QFABexA)z4~>vHFbBR7;{oyEpz^tqV1^4Yz8y z(v#JsYUSV%FV$4}IbaUH(mSfO8)2HQB9qf@T|RS$)}Y#U82)4UT&NW+Ma|*SqN)~( z?+Bi<{c?#0U$WN3Th<|?ne!e)`ZUWBtyjj!9u~sUPc-p}woGAzS^3gkE1Il>m6=(G zyWIXfzF$RRrVu|PeiRsI7AK|25*%&#;vCvBVC&r#(9*8=dtWeWL0`QwI~L}fD)+OP zs|$VxTd`9q4$RCX_}wik#U69avP9Kmk2o_b!!YlR zbUOVr2@JE+#}VENp17AkGJkI3O5D+copiGAH&rVOYo`Q0@mAgXVaJ~1 zm78(cE8Y1sRoSFD(>Qf_YRZ3o_&~yb5RXc-n<=0iXhuM5MykyZhUWBXPP$Yn*B2M3 z$zPfaTUc2oH-65Ib9=m>9{r;q)mqUmlD>FPHOGd71+w~nDE_SlI!j>$Si!wrboSzfQRysBfF}J7EtruTr z;);i0*Ni%)Bxsdhj#&?!re=4Z(9`NG2VXs0etR?n&tVu!9 z68C$0AKL(D>g^*V&K|o}1aDDgW&0-YWY!U|C?%da+dA&n@&-4oE8u z7tfB)bWhHVZW|qTr(Jj+<&4%EnQZoD+775J(rv|>Ck4Me;Z;{6V@cl% z?n+50osYyFOPxxP$?I>!7tg<*#IpAUpF{tJGusLOMim4!c^}p7?5QN_(tc0E(!`d_Uw5#^Y0)zgSpQF_hNy`VA``K4$0GMZV* z5iRG>w^j;M@y(|_zq3S^!CJ3b+|6mMR(6_rsHH2_WyqVDmONjzvOCIHS+?@DZDg>4 z1)wf%_OHQm*V-N%<5=o%hNX+0>{yV;@E$DldKq7^YdNdlJ|TdDom53o1{JVHtkuFV=&6zAmUd zCGwkm^)bumdEQ?>|CGmBIWjW4%UW|kI?uNcO^Y4s$>v*KFtY@45+nVq`8 z9b)f#3uOeaSJJ8$E{k))UKQ#^Hja1!Unpp8{t@`r+WZjz%d9`=TUM@srr)*A3~shT z^!^>QGn`O3!6-OOZx~w)L*M8Qqw!-6$IqI+$5Px65V+Znn7(d;NL8 zSAx;XvO(Sk#wu88D>Goaj%=692yxOYWOkD(#oXwAKKdXhF_?X>_aM7LqlPe zo{7HtaNX~`7p{1)A_^YDbmwtRjk8~Hw+|23r-J*b^5Z_o_qpKrABcDV%G01()a-hg(aBws_w)i!I)f<0>G~u; z%co)(MM|r1ihMRKH})hfjXiywGj8cebbT428I_EZ|C-|l{4dYbS@Eq_o&m12W;-Vu zv(t533CU$&#;P~M6US*`rm3p(y*KyyJ`+Cun#RYR8=HDC^ur&({*9Xg@6G%_qok&` z?1wX>a|?ddYdV9lmaZ$aWPEEgGdVmtwq{M&CZ+_2w!7P|w?xOf|7Qip_q`PsY0_x{=e*5kW3> zoS(OdVVByz=aGt-_BCsQY6dt;P-gj(GYnu=gogHhwqh!%ertT8tLsmwA?(|E?xS2o zd(Y=R)2!*IK~ul0lor}f`BmE~KJ_ofTgrVvMaU9OTc20lBTs+N-@(YXH#Rl^D9g-Rec6?6G_4p zbJbxhQ)PJxBal0t49U?bCd;oBmMTYd*o;zHrhdE>|I#$K@1mWm({wD`F9&zMz|wCF zv#^M3W?o#QokMJ>uKv6dxiO4aDU9oU&Iq?TBll}~iZhqek!Y_LjrY`Rf0RF0K{>21 z50w|M7HYl z_Gp`stxpPP8&SXB)uh`uAusCo&Adn53b(qI`>i}0<)z)8)F$axw6}=H`+T~68$4OO zyuRNK=KCn@`!yd@#(7fdHGbh|zp_+$mTupH|M)BDcF*ZYr|-nSbh-f{c}zeQ4Ziv>5-0v3+J-?6t`pKfHJ;-Qg zv=~OyAAnapAC%uBKX}R)Xd0nP)_cTTpea9F(e58;E`H3lJr_CpDZC=>`6aZn@MNp9 z@T!$NqQ9UUr!(Y#lDMYyqv4!x=!jZc)^lGM&ZW{<9`W1a4Myeq2bc{W3CQt^JDRg%(Ed zw$^?Pw`#5UZLO928=_Hbzs`UCVlB*QrWS@;`%QRNYriGGq1GmT@M5hM?W^J~&}MPB zqN!$PGMj6yr4?&!E19ab8O_w%(zQuH_8;a9{`^iA-7TOVve##<=|3iYS)2;`q^4V1 zR@1vnYWla4WV1Wgu?lAFNozCEwQYAKSqxO0hQo1m1jYUQksW_GVo5Q~30ox7Jg$RO^2hcz@3SBlDV7g|`~@ zIeTjP%yp>k@8YqwQSSfd$?tPw?+}=+ z&A%5wz z6mRc0+smXl*u6|kkx|^sbT6>tUMA&%M($C^>aPitg_oYAdvEtL9Yh#S7Mr$LdztPH zE&k$Orj~6lBcL|;(;lYM^FI29xKPIrd;Du^w-3RuwY+j4%2OOA$^v9GQ^SJAI1E4H zIb422txNpi$$COtYh@K^s;!y7{75wy2dqmP#bS}=CtBqD@E+3?$F2D#QJk zc%fz=4Nfhi+u;4**Yi1d?ESkHa53O#DLJWqL-&I({uag4^LG`B>yF=Q#+Wvdga_{lPA~_o&7D!{cmku!YCtj)0n`2qabuZ)(yo(a9> z6W~=Yo>(Re+?EpYrEtjG;@oORE3A$d{JDh*FwNyunpW|0b2&_4gk4^)H`LHsOdhba zF3KB9+=EC%)YYlHA23H4VK0zYksl0ZSCaGTSy(v@x_)~Ruf95`{WSEEc$^E$5F0ZW zuhk}Ar^k4O6=}um46vrWVlLnzF+SuKYlxLqPEMtAX5d#%lFt3ltTMX0c8WJk#4LmQK4aYg2 zd6+z*NQI-%M#sVpF8-|QGicYmN3Anudu!H%=fCu=x-Wn`)eid4l--P~dPNt2#Z@p_ zIGrk@TL%5K-sY(hBA^crr?E`B&*W_4Y|4jgQ?;6e4|Iu=FL3gzySosUZuU;*> z%KTR=7lG^7OtV(okEkwySN=DK|C`EWLQmG7G%4?WnyT(S#_!^UTOMxy5gyXo&%5F} zz>{m^kt_c$eWYnij?eb?z3j+$Y2b6Z!^Rg!T9-gq`Yz+Y^pTfq(giIq%g{o<=@CKJ z)*vgVcN=!dABEMF2k;Go>!%jo=~B80eui53xx)C7{$lz^cu#qcY@7EQX_e}lw5o+@ zMe7YReI6N|hB%6D(-rwf!S!>rh52}87W%?4(&~nLo>s&4<9gr;BQI~SbB&Zgt4ln^ zQJ7+AM`3-qi=(j3U~zDC|+Z7e`?MOQRXZ%Z|e2wxh7< zwxh5|!(&Hb;wg^8hQaJ8td2XzHIBlzft8KIwu2W(VF63=_I|TbnBrhZVLNaaM`2fj z6-QypgW@Pmm>q@5y|<&VtKe;F5$KC(6gK9^e|>+54fyM4#T;mWa-z{^%aCG)=I z-~*x48Eg1H-Gb^92QbWegkw{+MXf)P{K0*)yj?F z`Zd!`DG6VyUJtKI*fYcbo62NDx1~!biN$o>3V(cexYDGOYH-V=G{k<$ZM-Yq&*guy zA0q!Qt)%byIX>GK*4iXH^1UGNIX$AaVn5_|=t}Dg`7hn%<(f1?%gZvf&<}Z0koDpq zOL9{l*sz0TP*_d*0pCl(^^?w=o}B*tjPJ`^_<4o#qcCIoUj=UCV9SB}A*H$|t*URW zXwM2VeI6N|hIEN;(-rw%ALEF8Sydaa%*w0|{g9jC_ALf=TAO5j?XBDb9{L`E;Ad}z z7yBEEiS2JZ2Y0c*@jNiw-;g`@H}2p)T8h};cnMhSZ`{dyvA+?p)NU(Yw!a~_?QcZ4 z?Qgsc9^2m#PqDx8N-*2scs1@A*Vx~94Om%!JNY7yBD; z04w%4ln2HBhA`XTkb7_Y8*hZS$x??eSl{HrQg~T^BNP(b!|2VO1y7&i&G@r@3bkg@ zf~mD~?<%#(Jp}imJUQzWXld^ieAYS&w=&W`B5Hd!16yko9>ZO&a7}9<_q!Et&B#8X zdGFSH;9-AJHoK)cOXlr3A*}XZXz^$Fz2*6(svMG^A46a7L(c8QwGUykQV+sy<0|(D zcye(y^Jibful%_=Ka)oJ`>^~mm-Zq4>!(webnnlz@Dul^eGnQkBx}Q5W3UY$0gw9* zmF5Mc`K};Gz6)v2{D&kbz@|9NG<*~}O7m67S=v`N*Or?H?Vdzfb|iiXAdlKm+zx<` z6Nc5J+@Ijd>5(&Ji~9H^Jm@1YUHMKtmG7T|9)ATL6#VQ(Jjwr0BTu@xT)GHlKgAWy zmVK9|IrzwL?OJfWoT%m1YD)QmtS=A;t1G#`$dl6*eRJWlxAf%_KVOj_ zTi#A>*6)|#@#W3Uj#a*jJC!riqH^{%u=p$Jc>c7Y&#&WG`n&=GQJ*P2U{;@QXU<&4 zm+kp7`{B@!@MueH@9evT<#F>pDE!|8Un#kv75_eN#pQ1PXFY*x_L2EUETsMUA%H7Q{*}Qb zWvFz{-7~CN_$%qa*f6<&&7*N$T5q8o3XkRRx4c_j|F`(jh43g# zy)*gwy=eIS9sl)PI``i2MgMO=OuXER+8@ATFW}*%GLE<0*sNr{rIt=DadF(H{-0Xp zR^u(v&92nY+W1vBg(tm_KZex6#@8+TsP%g=Z`^QeDA~O6p9ufY zJP#rKSmRR8q3tZ^LaxlgSNhd7mD^oK;lHAFr7REm(nbmR8{x$g@OR!_PM5YM>29KQ zSCW?=7i|(fySvCUr1S9KgVePiROPM-a$^hZ>l-{waVd9PSAO*HAjF; zrqem~Hm7q+vZZCCSIwfUccyY4*EyFWDQ^;dFXBM(beEGaM$U%c3*4C$^&3LZ_8>5g z*f^miUd7{h?+xCei_yklZd@`tvm>4I?Plr~Myq?C+Mu<%=jmV{9_BkPefm-Mp&s|3 zUxXwcjAA@aB(iIi^kv{-;P$?zWw=}2*QE7G%@?YzGWR6y)jdk`r}@FIi49{Mx2kpi z@X4P4qE9`OwFSC#8^hF^s#R|A?CA2lqeypctG~bLnj)=m57N`2M=fgp@kn$K+vxZO z;89>r7KrxdNOqha4K4njUo1&7J0#XLLB74YN2f7g8SX@yb|*r_edqTf{9}0hy-3xz z`=O86j@=L3*>`HsvF2Sow(XGnIOzGg%vziLTkCg!@dQhh`KeZp2iLEeW-XXsJ6!>< zEZzgc{}ak&LQj^DxIbR;okKqFC&M4#9WE;jeu}{@kJCZ0)DMD3@jg{y7Q&E!msVND z9G{&-^WjIn)q&6H4y_mGFIPcVT2JS{bd#5B(g-at%h1C7Y3NM?_EEX2u@AONjOl(@R9}UpwJRdoBe)E^~ zPMmdSRHFB2%`b${)x1y>F2Wsag7~`RrvJH-|N1r43RZ>n%1!Xfl605J0#8<0wF-9% zhghXiSQD*zzoPY*`kz>BH|H})T+P110pe2JSD5)ro_yLk-<}xoW zO;@dQS&~s}bKjH}{5_(Dzpcif`_8xE?@=xM4Mu+}{NBSYxKV%Q^6>wW;s2xguY8k* zRGR${&NlzH^zyPf6mQ8&#_7$+muxn35^1i4PDz-zQ<%}Zj}Vtk?}c)Og|)QI!hXMG zFJTXoeC*V>UR>$L9oBy7pYB(=7fPcla*M!MOpyxS`+tX(IXyDli#FQxU+K0nP>7(Kgvemb{Hdk)L? z7Cr>LxVKPui`u<~>O<=;Q7<>YPjFcA&(Yi%rdkItSm~{O5zN-AYw35EZ=lE=Ysn#ahk?#qC&*>4Z75l2!LRVVn_%Ge$ z<(f1?%gZvf&{us@koDvsE0>2h?9h@atfu^c@2TMWsjuqv=X&Eu`itp* z2JfjHAlsG$^;JuCOxaYO- zVHRQ*zR+iF(G>W;t@w(4SH;x!U3cRy_FW$bX8W#k$G+$~0rUhKODEXCXV z&HAp2gYCQCjJw!(y%nt3cU2w~`>w)l-&O9t?Ylk;-X`l!|3Q7%XS=WzUe`X2>CVUy&eND?(Yd#>3Tk^^u5gj9Q?acK zdyjmY^M#L(l2f=Ep~g78CdMtqAsfBsb(UsA0-xO1hPc{r+^(%TAGbFaWZSt>>qoPq zEAmR}hRAUBfG2oM!Nhw=tqddT|Anij=ASbb5m)^@9MFu@1MZG^FztBdm}#u zZcFc{ajTSllK=N5Z{r@i&*0Wi;mc19U;T^hF0XtkQ@+G7Wp(^}wN2@WwdYGwR46Uy+*{XZ?IF`Uz)-iC0!D`qhqU)tPTNS=B>oxa8mI583S^v-9FxAuRuW za0n~N6jmnlyYjQ5>`c)a_l@r6nekvH*rgETIrlDVi&m$7%x1Hetd2c#taj)(3x9>+KRF{hR%td zZuy-5s>bp{>{-B96p6x*utuI zi)T0O5e2uu}eEphfwaVf+7estw{3^?d zPtecxMnCD^ra=1{{g z8XCUJpV9j8RcFQx6Bb^3{AugjnHr@|DjrJ|kL$9q<=$U$;U>YPhmf|6X3|Ev5FRb* zfxNGr!+)@(_mZp<{|ABVS4O*6)PIvdljke_48j4(QNLtq_-z@O?; z(1D9H;T;BlN++qZ@hsA8Jd3oH_xh@U>BYlCe3pg(M<`yVd{NDO`J$fK@Uy4# zO5TC=P3{9N{*GymMHtnX0%=PgmiHL^pL1Atihxq>EYhA!_UWrzluvX7NKQ?lh= z&Qn&hGg`i63y&rH{=D0geY|9a5^DWF09?N^nk_LZwV6LtB16eO4msjk5&oYL{+}5B zpTz&BFyOZ(Tk>PcK85!b9;veNEYfT|i?m|Neqe~tgTnt)`EN`1ai(P3@aHetCnF_S zc$Md&WUqvlFWDmclKo(4#gcs*Sg~ZU0&7*WS3`@x2R6r|v}C7}N69`N|F&c+)vjby zTGS3??T{&L#ujjPHf^DAOVvYM721psU?iLzY}V_|`xP zQJx#p4Rb9vSFG2k8(SM(Z9;FC_MWo2neevopfcAFQ5u|;B%hRqsB6uGxp%V{Eq)8C z|0Q@xAKssGmW2oqlrsd7djw_$Qk(eH~>f#;b0494-rw%?W{5yTzsPn^3UVPrX|CN4jTK=1g z{6V+32oHYUd%)A&9p&NK#;?Nz|7|8LdlU8Z48OgJ`gX&g<{+v28ehb`xWn*h=`HHS zmVuWf`T0AzUK;%dyKyIOJul<2cH`x8lP>W``T9;dzFS-OYQtk!#a;`(GU&?kzh3-7 zKkhy7s7Dy(pJv{z%(q0D!A`stx1OSXe7pQBjL-(k?z}^M*pavKUq7W!@?x0fcBO^A zF?LFM<8-%BQfrm>DI8>5dDf?*_s~a>`!PHkAMaKp#7}v%t3_G< zfst#=?2ioImFirzO{@5^UjEeZ{LH*RC-K+XE{XFC#Nn5bC&X1Vk53PAOihpj{OBr{y;9kV zJNBQKPth0(;u^n3eAMu-H24HKWsEIbGXs+bpY(WG zcp}^o?}oKr!>i(nPxkAH-oG!q#LZw^hQ?;=vHS~vlIL~XB^(}PJlW%2 z6Pm9N*mWLT$Es5b>w1s5yGxW_$rN8#yje>1*!b^iIPV3_2cPFt(B7}w>fiftQhgNm zheFuG{+MAY{lfq3ajgcX^b7x=4Bs#|w4HsGwPnKp3XZPD@A3P2{BNFLW@O#$=-7x3 zb#Z#j)5AtOMxc&?bR3=Y&sLF{>B%vPZlo}m&~}9|b29Jk@s?qV&V^phVWM}PK-0%# z>wvI+VX?diK4B!7i!;66#kbjF=tkIOJe1qcv7xC3Cex)&_~jmFV)iU|s^xpB|dh?Lf#B{Wed} z%goa{hK6_4dxvP=R1O2}1^7eHeo@&SW|+}ln8~b9N7@P>-itkr1)b69$%%1JvkbSV zN%jez=Orz8+~yY(hnIOiCTFKyIf`{px*F$6c|)8J$Nh27uhK*ojZZ~>(LTw84bW{8 zu3ml(_YVu$?{NQ-$Cd+Oe<;EJEX_(jt$)JU;-S7YpZXsHcn7eEAn`j)zI4@zG!hg*}O?3Ing1I-?rmn^ zy~Sg?LEDuum3PT~ZXRdL?kth=1a znW61&ivE%_a?o0_`8mVfdhNV-Y;PNz+@9vI#tCbJr(ffSxeu}zT3^c!Fk8XC-(Bv; zVQP)VoaWuxrHnPg4^%eaX=Dgzi|B47p>@$^AiQJe(VYq^kmj{fn70X9c|= z7q-V@RrRv}4PVdf%+#zdklH>SZj#@>@+bMHB*f_JyFr76=u1WpPeQaGT3X2|8jCFc z|D-Ur7iIs-ne^@lriX?}pN~q%(Ad}&EE&Ro5czo;T5V_twbx_;3sFJ8dEOKihie4Z z&^=f#jedWO_&>#{$=#3T@Bxb!q zSJ>-#nyeDFQMA|dSWBYb`?q+K4EWxI`$HB(uCR~tl*twCV?0T2;QthuZ#H`-WM-nZ z6aOd5XoUS!@L%(!_#jKz?|7QRCch#rXK90<-{YsuqH1LEZ=Mte;R>tKt4e7&#M^*% zlx??AAq&WA25S(t>vQEQzjV=ZHG{Ht1OB#CUJg)MBTTJqMQtX+@0@iexBzEu)!9Zw2_a5aLun8(Txwwb3%2Jw%yT;2ycTkw-) z6PMlOuSQq>SB*|xvSW0n-aj=otUN`IrO6LnDpwbUUm5&_Ce`(@EPp1eePSedZm8#q z&Me=Sq3O|~E5_<>IG`R_fgT(EL=W-0itt?-9+>Qwaa&3+P;$pJeACd(@D6P6I!v&y zV=FyR9F;NL8Xd6_=-|Snq2U?2E1c$E!fPXAd4nFKT$lDn&|{!Y6R%GCDK5^=>Sa3K z35{~iU182ly3v>JPrHrso_Yx6pRm`1=|-g3!xZ+~66_Tv*vm?=_kp?Fl)|V&^4?Q| zy}bl`i^a$n`TZbRv4?pob@xo3RDMZ|u(O249#Fs@4(4F?S?pBrH({H7A1G!c zL$_fDkrSM^&h|LU6Zx2vp@Nh$eSLj+j4Y45j1t6qWehJ0wQOOOkdqn9MJV5*jqzx2 zM?K026-DK{l*#l$<|6Rhv zOhdqbpYUvELHyV9P&d5n5Z@ocn=te#zP}{QnUkz{9^_|!+9$M=Qu!c%e{SUudm{SZ z5}m%4GgOnAY~mxm+=gDP{)9cd1iQ5aySW5=p0I-M?*MZ;F3b?Uumyg53;g*-oHV?P zbY9AQbV>s{6E*@?%x~eBi#Ns(Jqdd>*!nuAlO3&GMqM8nn9Plw#WN(H0Lxwr8w4XS zHIp+soaPcn>EQYDDW0wHY}7ryq09<<6j=YzPH)$V?{3^r@v^%7zEfsy*iEl*&FZWy z=>ILyOR+0`pXSBz#TgcjHSWaiZ#__d-nutV?xK+p_EP6((vA3aUhxm;%h1htJALK; zVQ_mNq=`N_J5Fjp%Jr2x`Z`Wu?Yc@r z&7{+&MfF^mjjP&sVbYD_o8kxW*U|4V<=&PDVN+nm@*un}UQ-@~T?Mv&a{X=wsHOQT zo-xPci;2pEupMA6%R`>0OgOujy&M<*JmpbgT}fEBTnQTmOXVua8V6I`O*dw0bj-CD z=^Sb4R%4KZ;IaA=b_`fiU&7nO8}&s#2s<2%yFa)9$n|twI)onyK3}~O&r#xu^(w?? z8JJC*_>KTeX@l>*U_YS@*mw*3Jy^ETZ#ZNrH6;F&`@b_!5miZjw z_6vAy8YJsiU?~mD{3_bLm?vOG?CRN}vGX@~t*4Lb@q@tQ3c)$&&^Cotx8uOohiV_b zdX|}mvs^tY(7Go#FsGTPJpg~Muhhp5t)T`)>R<4eXvt$nJ2@ZTsxythNDJO_vKy7@ZmQ-`R($pfF%${Nni@@AzoxG0JYP$bU3H=a*V&Q2*0a$y=-8z(2W+rDbxdty;u9|I;G@3r3FksRapm&x$A7>k#gzz^pA6-=}$! zErw4k8+Y^A_zC+ZPceSNzb{@BKVd)OY1uB&c8KSvJoBYNJU9`w34< z17m_O)5g4(vOyeJikR80$wsxvu|F#Mn^ktqiq1#D+8OuSbPMY)84C;V5U)wMur9Ec z{Uy?!c;@NTi048lC+kxQy8vuHof@8&eJb%>gdD&66WDsLxRQB!H-)Kwka#-bu{suZ zK3LK*e4ht<3Vji)FJZTW70Ze6r-?V}EBL<&Eafly5#Nnq^QBWf&y>G;bSR$d;jw8D z_6)F;h7I06y@PSydFKrdcAnQ5?4GoPm$lN#+O@0FJu@7ZxH0Z0R6R1AwD3I40QV}c;7^MP30h9ZxI&r z8$F2ctzcP?gs=kd+rVvF#QP4gRQ`kicY?Vyh0Gd;c;5r=?L^1q)NZF``gcCe4`eRCp`T#iEa#+ z>rZ(4=MvrZq;j&qlxQ1AC$6ebJEHOXwM2K#hW8uN{H;Xm=r8)aJJANXG1O60K70=v zodho{=v-K^d%Z&q<~Twa!d-2Ovs0l=NiRQ880sRrtrnR+Z2q}^$HdT>D@V+O%HNOh z=jT71WX;jcHmrQg%janezsi%!8}Sr&H&4;-2>-fxV>t}|zX@jBQ}KO^CzUtZcii9R zX=%F&S358#d_AFRbfq1J!Y9FVV?n|eKIQqnjsrFw>e|rEkX!E%Un9fZW)aPw2%iC` zx0?>ajd+le;X(E+{9TQWlf$T5^-VibpC%b>HF3H}uCVKQE*NTrW3+tT)*9FWw53+p z!X6A()V1&f#T#{v4u!D*Jg;pQ&oal8+2#V~Z8LRB_z@ZJrv8qNw1XYm)Z^A;3jQ>r z=*5Qml`{5OYFEx4cv3r12&-pi2NQ5A;~`xK5s&%WF8L2UE!!^fIQ_KhAA~d?MOYyf zajsEmXYAqAb96KP{EiN9%N53bn(58Dp=^lvFnFyk7Ir9DsvDFSVJpCL?FM#1__5&V z#brurYeGDiL+h*4p44~NH%w2C`ra03KJ0Hq4HV zwNJB`F3YbIBaIR<lNjxSE!rlNjUmC>oI(UlleLa{>m*(W>cv2b&Q`nP) z#ki86!mi_S>weCPnEXVq&2(F6d>SMQkR81xs+Qkq#m-J=ZK z#*;?zpTpzlr`?uWm&Wsna}Q6k{S)5HLmA52Kk`v{pT`F}dReIrw>S1`ax2tr@$`G1 zK6fxU9U!Z3-RZr9$DXh0`mJCu^#0OrXWj5}2M_IN@6hfstspu3p&PPZ=Xo*XGn2!U zV_P&!#e%V;>t?LCdwRT5WakhQ|7=rmZMJT^`G}{lcYOraCEVXF!S42$?=z4uhwz2V zQ9S6TZ)jqqi>38#e2+UGetWKT6Qj| zUvi6mjwmmqql+L@;S}#CSKA%^XnRMi`^i;D#CM8|7X_<6%A%dGJHj6bZqGVZUJ5@G zJhZV+=fYMMFnyJyn@N+t@T)w32~QEb4J_TFM4ZI;T%L8)6pP^@K0EO?LS^H%Jl4O; z!`lj&PFZ_TA#9!AtJ|@xU)`*v8=Z<+?p7z_ts91P)0pKGmb+yPK7DvMkc9O>m01I;cEQJ5i)l0lz%c6ws- z#J16i(Z-JYNYP^)cPLEh#C1VOA5&M~AYPaf+hOs9@JUA&q{@wG_woK+IHCNqWewI~ zkHm8nJgJXH{aegIdHmenK;4hJ z+@z7~3y6MG6P>#8*6bQZ`Uu^^EeQu+Xirt9a*fhW)hxc zOzJ`PL-gw1*RjF>GYY(#Z~GbJs$ANTp|Eb`DbCXhzlq1s(x6-NPuMLy8iS?Pi11mK z(kc2gtsmsez8puM=EB7`iT6c3q~Gjm4x1Q5UtK(};Axs8R2idxDEf1GD4&Js#8oxQ zP6#XUxg&7aZP<`lkVSq-*7JCXkDI1QUyX%y-|oY4wB%LsT+>|tv9sCN65%76)3&&8 zC-QZ_;8%5jul*L*b~?|?e~poXJ@0sX@DitXb?Q{Dv`EyvMsyG5kpKGjLf_cLZ*o*7 zB9_XL!Q!_$md|fpMz53|`7M5<6M5~oFp;CZ6!EFXJQTzQ!f~a=#OvSU=-e@WS6-#DmPcz$wpuJQuK!fTb_;!T!!#;i=3s z*svoOzXX0JVcbf6w>%Mx@ss;Dc#828{_G--&V;`doVj!=3H#5rHA8iPw2JmJXv95{ zq$h)?G-YzaDFg87l*`BR*slzKg1ob_9X1USi)oPiVek~wAp9cmVtfpq;-hbwzRF{5 ztFUkIM7hzno&6nTUUg=1}xPT(xa2XPp4hZ z`49N2B2GNT-wvMS!uwvZFYx4Kz$^T2p2Q2bhVVD@*fd2frb+J0c#86bZ|5n>+Xj}> zL>NC}uHf%HDU5&}Q1Gjh-5aEUiJe0@gp-#TjX?+%x zEqfLA$(CDTuP(t}Q-ZxtSS-uvT7KUP=Gx(4Vxql$KX|;oKsHCb9|nthjCEADaJkB= z8FLCd(UxD*DC}&oBoj=w=_%L_>sQ#*O0XM*nYfGZ1z^Rv3m0#^ts&aQJHU!zzXUAa zz7Xx|onW~%l8>^NkMVsHSHI_adPub(^u}PLs*i9wa zEy4+`kf^DMP~kAk6p`_8*?2-$Q5R#*G_k%xJIo*rTD0jF|Q}p6b6KHV?CLb^C*ve6u zJN;rxgD+Iv_bER>-Vb>WLZ15JisO2oO&or&vY>W#6OYxaupVJXufiV1gU$1O!%)6{ z2qwMS_EGKTM4>Fjw$#cLW^dh5e1zFsb`Ib8heAK2TMq#M&mNPYOoYwbnE;_X7Q&+t+8BgWYDZB7Fc z@5fzO?2GplH?E2OhGHBdZePmfL|sd>!FskN;^ zjyt9GQKfQXT*uh^dci2u>eItJb{7Tjfywt_i1Wi^oH^C=oEYv-e`}9)#;(uN?VKdQ z#&7OCUEps{sZ4I$)}W(}ej~P|GCb1bQHMPYSqE08^66tvT>R%kYpYCkC;=wkE^qdu z?{yyZ_&cmJujcnIZ`L%OSkC^Xk@_}1(;itqe99@yXULky@{@+$$F240<)ixIiam8F z(^p%xY%TF==h;Q1mR??#C&v%w6MFn0=W$FAt=m0QZw#(_h>PVOc$D|%M1HsWOnIp*)ZU?-#8uWMD&Ff4%i6!vYz-YsZ8iF&_61Uoc>nD)xao z&_!C}3BDKd?&==|D@BQg>#&#y1 zOdP)ZeI@M>ZFB!_KI={Wg=3yQD&nQ_;`Zp_!5L0&930VRKG)#oARjN} zKRU`Gj=FzNaYPk=TsrqAPR3IX&wy~6Voam6RvN)CtPqBd-J2TXQyt$D2w&pyF%A`| z4-QYcRz~=N!d;vILre3B*U3M@vmSWvO>KUhy*uI^p2U;e(w7@+z3)AV z)qByCSp5*RVtW!4lYQz*j1k9TPhuP_{*If!C$SH4T(oQgx$QizC$aDP;6wuz%=vpZ zXBPCqU|^s0aW(NP>SG!#{*IhqANvr;MavqD5VNs4ngG2>T^Qk}i! zAcJ@^#X+uPYkBsZ?#(AC#J5M?abhj0fs_FPgXqcZoOJt=8JPhUsp;AYZr`fA2d zix214Qg?##_2k4JJgfX&a>gE*D1huQv*YXBO}M*Ql9=4pIE&kGN5fUPv*TMqw~vgb z6P9lvF3;fk7l)s!5BoAYl&%lE@3m%i>^{t6fmW|B)EcJF_Nms~Ok9h#<`%H{yWjk^ zW*_3XXxXjEZRc^dhIuZX*YQi9rkey=`#OyXOp+M?t5u$X9`%bi#2rV|cOcI@@E`AS z%=i~nZd>ly6aON3^|a!DUgV#0C&ooAo|LN-T`ZiuQLfAWq zt^979^e5U0yBogHCb`p&g?%2|s##Hg>A63r-{9J^E6~aODXYZkTY0~w*PU3G!$V{x zLwNC&gBZW(klJilu-_3ZKT}y8b^n3?STx+ZAo?tHR_m0sQhLMFwP@QjF}9od?O%B= zVa0UE57C4_-{9Tp5JuYel?!!MPVM%PBNcv}YE6(`J*viA{43=i-WI3oZG(Gsa;yJG z-PFw23y}9h9@-VxPf%%z`5E(A{RfN3e2#cAuOnW}?}!)kTst8pzbxOQpFm4_ubsg1 z(<(o;gHrVQ@-Fhsmwz$*VjjwhT}WB-Bk@|>hWLIDEZS*%?gy+stzgj(+eeG`dh)nv z(cSqV0Vg7cF}ua@%>FMKgUWH_I0KRR1Jz79w&((5n+C^))5m zwKjB^R%rDT-aW}l_ns76>O||B;8bIMcc&J70edlmEbgiq9iJWdvJBoeu~SDQ(=u#W zeE5E$)qE@Q9^BmD**DnUG0?T8Yv8iMUiuTA-5s62wQNBod)zL7x8eUX%EBi}yvp@Q zsaXDmq{6C2IHBwBMO(*FL(F=#%O=JC=VDgJIO5-*29M08tPuPAjtVj)HPEHGhVXZ;-4hoJ9qZh`p~%`JS`@rqwIxqO<{sjQrYrpOcG^^xU1UT&s0E`3abC0?SGvuFeyy9oK7J0ZA zA@5@x7cKkba-%mp+h01XI@r5uz0VC^yza3-norb<7Bwy3u_SNetoFhRvxObu65J6@ zVmI9U-)Hc*q_T@I)w30}!z#@)<60Fva-LPO&q9yCW17p+zOQ^xrap(vcAh$d?Mf-d zD&;4gMSe3o2OD*FqOZ~`Y@x7Oft1gc`?+0HUtBI;|65X#O~Hsdb%uA9n|m-?XmI$U z4*OL-_|kHAX!5+4V3zjzywRP~oTck#*JT?ON6k(ztn3P4Hn+Q^FgLCF*#j%P(kw9X zbE^$sC5-rE&Ur6q?iVfl8vZm7dY9%wvB%k|8wGnD+PZ)EwvI(y=3D40y-aLx&yHBcgW|LjpHeGk~SLpIR;GYw1u%Jt9 z(G;>*@bKHG(cF)go3@3#*UIUS!QyX8rLef`%|qS}s9dAhpWv^Z$F&_{j^l2%v2;B& z5j3c~!$^lta`LA`719yf{hOehQ!wL+nw$fHf9CdXMt65IUWzAkN9-2xj&Sm}`ljNG zSUTN0#Os@i*RCvLir3?j>A%P<#EUSsyKX6uor@H9Pzlynf*n$V9VX1oV@lowc-kA< zb>Pz~?1U2R#1c&M((XFaFTW@AF#gK!5DoYW@VL5nicUrj`2Abpr?kKyWN>%Fa*%Us z3;Z;2?K5HJzRUOHd^}f!`_o0TxSR<#FnLvdf-kEr?^)pC1UjS0%std=Yfv+$cI-1Gi8{jX)rHhDnsa+_MV{!4jFF$+;IsP&{&)=2Oei3rw zTtW=98;ox?yF13GlAiy7-Kj7*vSnCURhTJ<5!*<+6ZIak%@ec*SCL257o8~XPdG2s z7t*cW*w4e@X<#3w+FkmD-wvLegQtE8e@WnvVG4U=wJ4y4Kc$(}Z zw4Z|isf#pL@c*nr)sU+eZ=gFN)!gyZzBoFf<;XIu7c!Q_YtLW1_ZiiW1+h8nnK^)73 z_V!5fY}@ww%&uYOp>J>Ajb5THSU$qZ=3K$4`@GfG>US{oIJf_Rjl9}5-06m_$i-{13jIUP}S-aWZX<;my6 zS#(~f`09RUTZRR>Oetho#fKI8UG@~|LOxN3_9Gr{E+>?iCAbH7Hg@@N28Rd7kb4it z%BP3XOujRb+a|T@bl11Y`ZxF5$NtT^l-J_e`8=@rJA4mg=e>_3JLQnuzC6xW@870% zoa?b)@uzyEpX%0W?yXumKy?cpFCgEXj_3Z$sj-?lU5B*wlT7O91}kcsuqs^PWHjf# z0qNcy+D>X+aNAD z;eT3V*?`RRd023BHCef0bty|FtW2MpO%<(GdSlJ|EH#RKIWk1f^65s<+j*L5Z|H#@ zlB-3kL4Gu+y^XXNj9PS^oZrFX_dD+JM|n^d;;4H`Wn@=iqAhci$&_)%St z(Jgz(81i(m9yr+5%}Uv(_5qEww56bofY!;Q4g0v(HRV&`H_tSNrnKy(yk@k}OI$AI zS*1L=2i9ZNURaM+1?$1LPx~}F8X$f}>#^DB@X$S2kG+rMqGel<+rB(l4{n219s3O} zmMr}2c*y0+X+P7s!r!-esfkOKzNgy}w}t;o_X?X=?#9<)ZX?@TpV+A<)s4%LaXHUH z$_L8&uL-kdxci<@tM|f!t}a@6s?t8?)1!!gF`ovFZjPNlpY|b+ihT?Lx+vtJ(X>h9@txYE!!xW02^=RoKBK}9`Y+U(b0JblYR`?`&t=ePIf zIQ*`SkeGqXdOL~BrGp*oWFbZSkCWZCaZ~5O1wHEr`#U=Z2KzU)Z`|nkl$`#q%R7C% zs}-M7ip{333%EgAe)_hwW3~ORjXb^cf{eDL;CCHzJJxON>9{D$b~Q#9P#b%0QCY;dIc` zKY-%>W=cqrG1Na zSe=`1yowqh<{KIvlhYTBOk(9>*K>rtGYFe`jMNbm4VV zUTv99HT|H*GV!ajfPGk49Nz@&<6r}Pfa=mkTy^8@W%F3PFwg7b6XI}R^?(LPC$AXX zrenz%$NDkMS80LL+?;wHAx&XjG>l{XNvemVBO70xu6hIyt)@;_<%?!7b2sTXbBbXK zDo!mDb_nC&=#4R0u*}}?&lDdw1#ads1Ha~K-4w0zH;gl%k`F808sv0EUwgC zKNj~k7BStJJji&nT9mZCeqd5t7zV<=iWcGIZwn$%bTjd)vOEcT{)N&te%1DIpDb4% zb=&iPWqug5%+k=v!mWySN?U;@8+k@oWrc{PeEfxMd0BmYYI~l`_UH-k%$`xK~U7C8Wi;C3KUw z&N$Yrk9;#SZ{dl3GikV08f8sHFN?cIn13d>U8&>!25RpC*ZJJhC{^y;2xn=nR(S=v zub;lB5gxyxcpmhXa}IYo|9tV2yLtcB%I)C#HPcMaAHC=h`Fk$B`u~FP|Al2Tp&v|% z(@*Kp`Oq;PFNQz9J6vgk=T3uL9;G3s`(?ZQpP*OM*>)&CrBXm<;Cju*m*qiYaPcDf(_VgBEZ-XbE+#>foaL2gn zt7KvC1S_6IC;VODveNN@I)kNnd%ynlxr4F1N=N*9U+(|H&HS#uQFd!X!rlXBzr4N| znDRh#QBlVcTL>mSNB0ukY=#oV>`Jww5_=$6U%kQVlmGIRcvjzwyrxlI_rk+hRL-ou z&xNJ%)I_!_FlrJw@=@PVuH{m&e39ee$F*)P_5H~DG?s=2@1D$#td6@`=9ybv9rv+J z5ylE08jpP%b$^IB7DwG5Hab|ohf(+5$C3R%$ZcO9=T>X(W)pwwO`-oA=BN*Ieamc; z+O1Wr^!=?}`TkjJIG0vs+eW3-F&qBYpl)?9XJ_o%q7-L}jLw)sa9qx-x)%xn^ z;cKcP8y2#QuvYs5wD_x*$41qoEEc2_?Vnjbr#0czwKe>2`1El=4(X#e<>ni|NZ4QE zk@?qdAGJFx6q%~}aDMJp7=7^(zSPtgs&1b3EBIL`Y)DmByneUOS4~)kfXzgeXiBjp1w=@!ufUUb4t2r4U%?*Bwu)eKFaSx^4jfP^4U`t2D z!)PLmC5Ug}|2gHj(HI_@*akB#q1|bPTGGfXtVo3sYqjlSh_3Is+UJd5jT|2tZTPQb zSP1XAzegbod@wXe4do3TTOVQS@J#*b!xjMaTb9-lR`=woLFRz>zrXj;!mz&oy zd2%T*ZMzbezx6$1YF}sDKQDCc+TgL|YOcNUsmag4w@J}xZ{O0@(?7%4yyLW}#HsW) z%IKpVAA{YFtz^Qqgv(!-lo2%l5|gWL%Fs;IKEio)C7j;)AN>V4mPnfd#@ z_r33X@8x^eHsZ7L%FkX;u$n6SX9)anWyBHgI$3e1D3~#C+qV6S-MfATX2G|AX;P>- zIC>A9V(|#^uXlk%4FCGoc28MthcrDb^5GvQ8~=b@uywc9!L@E4V={5vZ@u5w13&-= zprifsj@|A>y5|6zHUw0lPUZjvhU_)^NA^1DxS(YVSSPjp&!6o4+j=aNvd5yLRJK(f zYgPOf;Ir<&9)(oHBapJ5v$=L&=z64-vd1E&RQ@(0weG$)Lh2`=PIUvMhV6A~lUtqI z{LVY;A+ja@zs3gGo1X(@W)yYk2X4rK^#X5!V;$F{^HA_ugq)g>GD7RMp57L@mf>C* zp2;M==JS1kkQa!^9QJ75%_4*7^s6t{<6}>LEIOt&`}r-gJ@&g^>r35CIxx_$`TP{1 z@vjOrzUx+B?C0;U$K#>kv4}j>{5F{y*PC*$T8_TU+M0|CbiU_?4j8rB4c4jbS`Uv? z@>qPHqsBO|@(|k>)vLP|y%9n`1PJ{~fzX0mYqFk&ydIsk)W;&TruOjBz6E``;*K^# z#M(D)RUq;mw>o1xReC)n_TZ+Y!?()}49VRDt%bk7Ewlhqd<1HNz> zh$0=I4trg3diahA#hIeO!?^_6Fy=_h3vVHrX+|u7$yBdg(tkZecu}(JyYet9n-vyp{8eV(md-f-`WwS>$ zj%QY;d46g)jSZfAy_V~fyUCr;?jmq`dUuJ3=XY0eAdmrCLlcNd@7fWFcC-cnoNH8S zEn&mRR3UUY9sgI+soSl*&i8f!hX#e?sI8W9FYQ5_#|dyQCkkgU^2dJrkD_nXh{hftb4 z=vl7}B3Jj(dB@0y;?T>d$Za(>u1B*zWoFxq$J*-t4RShe(0P*P28tg7O}l-ydMhR{ z<2`3@Yp>anwd~vh-ZOW?>G+?f(OnNg@bDGaE#4vmt3VU+%Q@9 zWc=CdpMmu}GxmBN=cw0U-xXKVtZz@PrFh`XX+Iw`!s#sKE$_Ep8e`6@tPQxm$dOKG z*&g_E5x#J`Y&QNRU~JbvI~faiFt){wFZ=K3V#`s+GjL^jlDxJtX@%w!H!epWB?q3y zr9J7{+m=G~a+taWnEE%sl*2!(wKMf_zoB<_>;i8lDW?xSh&=G;N$b&;(%x{NfyO7u zX(Mr?dwQWhoQI1jD}bF?&sr~>W6=^n>uDWqbtTrB;(^a6ww}&XUZ(YQUggDG zPiNU4h;tFXaK3ETvmP+^A0TRU^mhgJHfbNTawW|r@p*W2Dd<_Kd-4KojnPH@X-IU* zsAR~KZQW4W-VWX6FNe`Wz-W}BKTLVZ=;wdh{-ryi&a~&_(^<~5(D|fB^_?Bg>pTsO zYvn8BT+jSDOILFJkUkgj%VE#j9`%a$SY&5!?R*Va_LOPQ!=6hy&(I}4aeiA{bklxf z4@a3#$&M!`H`28;=_Y;(CZ9H^y-)ma1*49L!aBU`%BM5sdDwK8@-kbZ^C~aC9XiWa z@@(hMMfeoVJv0ICd0f_A*!o0kr6hg%oh{sD&=BnlCL5 zR9xo8)21lg18=TUcM{f6eU2eTu<8o1c3Z((gxzvliw1APeyv#Yp-A$;no{~`*4FMC zAoQO;zxvn4?*zv%!3YCq360i**{#ISV%ts2a{WoAoje^A_us95WXJ zK%`EA2x70hww3>`*&MKV5P8PNv*HE-ZK&*ul4mt1RG#E6cx|BI0kjP?9mxF{Ah!do zrFM)%gXN#LHE&zCe+dp0b&f+jek(9?j^#Uqz-&F~0RCgLco?;Rf*j8@`0P@yZSeWE zFTva=z+@I)oA>z0-oBKxD8B=3ZMR~brYD&Kt9A1%j{H{0z--Ipzd&~_Lza7%VateR z)G}rnw@g?jEmM|h%YDm?W!5rhnYS!h7A;GbWy=H0ie=T}v)OmE-)8^K0h{tbM0K+L~yVZcAX1mAbS7bO1vw-EE;E3bL4;b3`Vt9Ae1U0Z+o<;S1z{&*{R zN5}d$z*ewN*T*}zTDO1R%Kq__uXgPGl-&*fw(j=uk&)f@g>6@YpDWR~A@E<@bUKdx z;?qxT^J_od^po$t;sqAOe-C_L1K%Hj@AtuXYVz-2_X!5o&JVf`orSKTgHQsz04*Q^ za58!TP9wtMlUNd*0~f#z@O8Kw?uTdKd3Z0fA4xzGk#ooeiBb zRWqswsulHlVvsH*rkG1aB3Vu}QI$k4b&ZHpHxYT-R$`iLCswFl;wINeG;w3ZA~8V( zl4)cySwXgu@#@R!E9y@5yt<0^r6%9FK@Cv*G*#Xenl8;!vFds}-)yP);qg1IByY5pizEu?Y>Ef2UD zU9v7!m!-?m73z9*gSrrXl>V6hq&`c3S%3Q%)%sTbRehLYN#A05pl>w$8rm$uhGNTp zLxUy4aLkZm$TDOb@(dRYb%rLxprOYQW{fZ<8c!L^`CMZWUtkR23yq`3O1_4#=Ue%^ z{E#rjPw{j7VPTOE5c-5DF-I&DyToqsp4jfaS4xl)rL$6*6#Qm`)GFnhN==2PK6A0D z)l_b(GF>s%n`%sT<|b2_xx>_A9xx4=hD~Fp3DdM`$+TkfH}5frn#0W*=2PY(bBVda zTy4H;?zNP7mwGpQ-|+78?)JWDDYMjB1}r{*1q;zP{piR41i=W(nap4Ob@x~P-|_bf z_lod}^h)wN=9TPq-0O^2sn>+pq}P3~882VR5Aufspg?F3v=`bBg+QTD7!(0TLD5hQ z6bl`K4nqmh5$GtC1RaBtq2o{rbP`I1(xFV~6qF5}fpS63JO`bJ@}WYg7%G9vpmL}J zs)A~vdZ+{X2I>NJw+HHl`k;R34m1D_LPO9PGyzRP)6jir2AYNDpm}HkT7;IMWoQNR zfqh|rH~`)Q2f_Q`U^oOm0Efb1a5x+RN5L_09Gn0r!b$Kk_#~VLr^6X=CY%l0z&ZFl zd;!h}ZJ`jp1ed{=;c~bFu7ag47b4@a2IGLeW0D(f$za1@F+Y6kHb^8Rqi6cUZZAaS<#l!T-psYn`zA~%qB(E4s8JxDLoj|?D#$X#RzxrYoRqsSOCj!Ynv$P_Y-+(%}S z1!Ng{fUF>^h!5(E`l0@40J;a=hlZdB(Qq^pjYeb9Lued&7)?Nv&}1|PJ&C5GX=nzT ziJn5U(bMP|GzUG4=AngXFdHOub{PP9eNF|M;p*avzz$;J zSR@vOMPo5oEEb2wV+mLymV_P0PGV_TI+lTDVp-TJEE_wG2^h?M}lQ;L-V z`%{Bm#cHuSn-yxpu4BzuD|QoW!`iV9V3oSD9$=aJu{+oxHiQjhBiJZ5hK<{7)f_gD zEn-X9GWGyl!B&CY3c~l{`|%L`03M2m;o-oJMdDF-G#-P;;&FI9ei%Q3AH@^#Wc&o4 zf}g}w@iaUg&&JQ-Ie0F97C(pQ;rVz0UWgatm+;GY1zw3)<5%z+{3>3H*W*oi3w{G{ z$2;*`csJgI_u_qcKYj4*9}p|VD&a%=l7VCpxsMDXL&-2QoQx!+$rv)0JVeHk31lLfOdcms zkSXLzG8Oa_>0}0(NuD8d$g`l&xI~tcW#nbD9P}QQcgZ309yv^okfY=TIZe)zbL2d^KrWI?9V$>R9z5b(}h0eOR5KKCV8YPEn_-)77Wc+3Gy?d3CY6M12YLSrzI^b(Ojr^j$UT ztLi3ov${on1N39J)VI~$>K=8kx=-D&9#RjhN7SS0G4;55NR`( z9F<3%r!G+WQ~^~;6;YR{GOC=aq^hWD>IzjuU8QQNda8+Pp<1aM)J>|L>YzHQUeLD= zQKQs2HAziV_o*3bmYSpHsU>QK^40ih{51iZK+PUakY=xDpC(wdUlXE<*2HS!G>Mud zO|s^MCS8-MIi<>ZBL1)rg^eH-p2IbPau#uBGefYjgwMNH@{f={CB9?xefu zTl8(Zo9?0e=|TD)Jxq_#qx2Z?A|~ibdYYc0XX!b5o?f7r=oQ+B@nZs*KqiQ>`WInL zI1|N0Gcn8|CZ0J0JdIo?k11q|m|~`cxx`d5)yx&(2%P&t*S*fDL62vSDmE z8^K1hQEW6D!^X0Q*f=(xJA2@SGC>@e_$?z1!O9J|0Su@Bf))>rGV4b%o{_iID7VcH1o zQEig;xHd(bs!i8sXfw50+EdzW?P+bUHcy+cEzlNfi?qes675CpC2gs;OnX^duC35k zYOA!>+AG>xZJqX-wn^KpZPnh;-qf~f+qIp*|LWEDY5Rc}HlQ8U-qnt2CxAb;q+QlN z(5?c%%#ZWu0=PhK4;KV{v`{XLi{K);C@z|dB$vviaT#1D zm&Kjs&T;3t3tT=|z!ho8Tt7DQ=p(&&_bN+#I*SEprdJ6)sR0q}!|8rwi7F=t6Y| zbz!+a|VfaiHvH>8`;P3fj}t2!V39(|C0pFTt% zsz0a?(?{qdfnOS{KctV-$LkY-hk8Pv0=(2TeFpGU&**dYXZ7dwdHVDEe0`z5NMEck z(O=YG(wFMXfFE0-uh!Q9Kekripl{SS>96aXfnVFEZ`XJ0d-eVL0sUS5h<;Q*uAkB` z=^yBQ4Z()}hA=~fA<7VKh%p>895y5vjv5ky$9vpx0(iZthBQOEA;aeVo-yPYat&u~ zUT}e-*l^KMYA7>Q0*|=C&SVfe<-X}AS^@s0G)^05jI+jhzRcv4Ii)5Q!iQ_K=iiP_?5 z@eGJ+^27^bfmkROi5EeHQz2H0RbsVxMXV8P#X9ktST8n+&Eie5UF;A$Z4pqPct;!* zhd?AWERKky;+QxtPKcA@lz3m973akTaZy|nm&FxvRrHa3B|j-Z+9L%?d!>C+xD*Lu zr=uWxN|Mr~3=l)*N#~^tQod9m6-q@?u~Z^mlrBl7Ahx8eyK z)k)W+dZ|HblA1x>)h4w|9a5*%CEb#`r5>qI>X+_F1Ja;$R~nM;Nh8vzG$xHp6VjwK zB~45Br5R~XnwJ)&MQK@jAgxHNl8@{w`^o-tfE+0Ak%Q!YaGvzG#l$cgkJzExB9nk$dGnxnCZT2jwC8o;)m%%H#5+JT1@2v+}&WATP?x^5h%AZ}z@< z`xh5X`KAI8j~0WdbS)aK0r6;~sR=}zsWrw^)Xu`(qQueTT~ivPBW+5V$;*+ zTyws;&=#YX*`m}M5U19e>&=bk7IV9~)7)oH_D=CG^0r2co%T|MEw!zY3X|vq2^n$qdCqDlGhC}|*v<+-u&+hp250wqCtgc%3^fCQ$ z`^URkjtd7HHms$vpMnjtcZ1`Wzy}OszOw06-ukD@huyYgC-%qBY-!&C|89EqqKn-6 z!>04uPFuQlr-)6j4$NB1Q+{kZJ3hB1ffvJp4&kkoTmQD{Z2#Jp3{G( exe ), const_cast( args ) ); exit( -1 ); // only get here if execv fails } else @@ -109,6 +109,16 @@ bool Spawn( const char * mutexId ) #endif } +// Sleep +//------------------------------------------------------------------------------ +#if !defined( __WINDOWS__ ) +void Sleep( unsigned int milliseconds ) +{ + // Provide API that matches Windows for convenience + usleep( milliseconds * 1000 ); +} +#endif + // Main //------------------------------------------------------------------------------ int main( int argc, char ** argv ) @@ -133,37 +143,42 @@ int main( int argc, char ** argv ) { char mutexIdString[2] = { ' ', 0 }; mutexIdString[ 0 ] = (char)( '0' + mutexId + 1 ); - if ( !Spawn( mutexIdString ) ) + if ( !Spawn( argv[ 0 ], mutexIdString ) ) { printf( "Failed to spawn child '%s'\n", mutexIdString ); return 3; } } - // Aqcuire SystemMutex which test uses to check our lifetimes - const char * mutexNames[4] = + // Acquire SystemMutex which test uses to check our lifetimes + const char * const mutexNames[4] = { "FASTBuildFastCancelTest1", "FASTBuildFastCancelTest2", "FASTBuildFastCancelTest3", "FASTBuildFastCancelTest4" }; - const char * mutexName = mutexNames[ mutexId - 1 ]; - if ( LockSystemMutex( mutexName ) == false ) + const char * const mutexName = mutexNames[ mutexId - 1 ]; + + // Try to acquire repeatedly to manage races with the test that + // is monitoring these processes + int tryCount = 0; + while ( LockSystemMutex( mutexName ) == false ) { - printf( "Failed to acquire: %s\n", mutexName ); - return 4; + ::Sleep( 1 ); + ++tryCount; + if ( tryCount == 100 ) + { + printf( "Failed to acquire: %s\n", mutexName ); + return 4; + } } // Spin forever - test will terminate int count = 0; for (;;) { - #if defined( __WINDOWS__ ) - ::Sleep( 1000 ); - #else - usleep(ms * 1000); - #endif + ::Sleep( 1000 ); // If we haven't been terminated in a sensible time frame // quit to avoid zombie processes. Useful when debugging diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff index 939133543..c7e023586 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff @@ -61,7 +61,7 @@ Executable( "PCHTest" ) // Clang for Windows // #if __WINDOWS__ - ObjectList( "PCHTestClang-Windows" ) + ObjectList( 'PCHTestClang-lib' ) { Using( .ToolChain_Clang_Windows ) @@ -83,4 +83,17 @@ Executable( "PCHTest" ) .CompilerInputFiles = "Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders\PCHUser.cpp" .CompilerOutputPath = "$Out$\Test\PrecompiledHeaders\Clang-Windows\" } + + // + // Link objects to ensure no debug info related mismatches + // + Executable( "PCHTestClang-Windows" ) + { + #if __WINDOWS__ + .LinkerOptions + ' /ENTRY:main' + #endif + + .LinkerOutput = '$Out$/Test/PrecompiledHeaders/Clang-Windows/PCHTest.exe' + .Libraries = { 'PCHTestClang-lib' } + } #endif diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestWarnings/ClangMacroExpansion/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestWarnings/ClangMacroExpansion/fbuild.bff index 3313e5179..cffacc2f4 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestWarnings/ClangMacroExpansion/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestWarnings/ClangMacroExpansion/fbuild.bff @@ -19,6 +19,9 @@ Settings {} // Not testing on Linux for now - TODO:B Enable test #endif +// Disable warnings as errors to confirm that we still see warnings +.CompilerOptions - '-Werror' + // Common settings .CompilerOutputPath = '$StandardOutputBase$/Test/TestWarnings/ClangMacroExpansion/' diff --git a/Code/Tools/FBuild/FBuildTest/Data/TestWarnings/fbuild.bff b/Code/Tools/FBuild/FBuildTest/Data/TestWarnings/fbuild.bff index 104c2fb83..8c81be8df 100644 --- a/Code/Tools/FBuild/FBuildTest/Data/TestWarnings/fbuild.bff +++ b/Code/Tools/FBuild/FBuildTest/Data/TestWarnings/fbuild.bff @@ -17,11 +17,7 @@ Settings {} // Disable warnings as errors to confirm that we still see warnings #if __WINDOWS__ .CompilerOptions - '/WX' -#endif -#if __LINUX__ - .CompilerOptions - '-Werror' -#endif -#if __OSX__ +#else .CompilerOptions - '-Werror' #endif diff --git a/Code/Tools/FBuild/FBuildTest/TestMain.cpp b/Code/Tools/FBuild/FBuildTest/TestMain.cpp index 4bc00356e..871420450 100644 --- a/Code/Tools/FBuild/FBuildTest/TestMain.cpp +++ b/Code/Tools/FBuild/FBuildTest/TestMain.cpp @@ -22,6 +22,7 @@ int main( int, char *[] ) REGISTER_TESTGROUP( TestCompressor ) REGISTER_TESTGROUP( TestCopy ) REGISTER_TESTGROUP( TestDependencies ) + REGISTER_TESTGROUP( TestDirectoryList ) REGISTER_TESTGROUP( TestDistributed ) REGISTER_TESTGROUP( TestDLL ) REGISTER_TESTGROUP( TestError ) diff --git a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp index 8d092de79..74925965d 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/FBuildTest.cpp @@ -280,6 +280,7 @@ FBuildTestOptions::FBuildTestOptions() // Override defaults m_ShowSummary = true; // required to generate stats for node count checks m_Profile = true; // Ensure "-profile" option is exercised + m_EnableMonitor = true; // Make sure monitor code paths are tested // Ensure any distributed compilation tests use the test port m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp index ef69b4b21..095f7c24c 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestArgs.cpp @@ -10,6 +10,9 @@ #include "Tools/FBuild/FBuildCore/Helpers/Args.h" #include "Tools/FBuild/FBuildCore/WorkerPool/WorkerThread.h" +// Core +#include "Core/Process/Process.h" + // TestArgs //------------------------------------------------------------------------------ class TestArgs : public FBuildTest @@ -26,6 +29,7 @@ class TestArgs : public FBuildTest void ResponseFile_Always_Short() const; void ResponseFile_Always_Long() const; void ResponseFile_CommandLineQuoting() const; + void ArgumentParsing() const; // Helpers void Check( ArgsResponseFileMode mode, @@ -33,6 +37,10 @@ class TestArgs : public FBuildTest bool expectedResult, bool expectResponseFileToBeUsed ) const; + void CheckParsing( const char * commandLine, + const char * arg1 = nullptr, + const char * arg2 = nullptr ) const; + // Some constants for re-use between tests const AString mExeName{ "ExeName" }; const AString mNodeName{ "NodeName" }; @@ -50,6 +58,7 @@ REGISTER_TESTS_BEGIN( TestArgs ) REGISTER_TEST( ResponseFile_Always_Short ) REGISTER_TEST( ResponseFile_Always_Long ) REGISTER_TEST( ResponseFile_CommandLineQuoting ) + REGISTER_TEST( ArgumentParsing ) REGISTER_TESTS_END // Unused @@ -178,3 +187,74 @@ void TestArgs::Check( ArgsResponseFileMode mode, } //------------------------------------------------------------------------------ +void TestArgs::ArgumentParsing() const +{ + // Build executable we'll invoke + FBuildTestOptions options; + options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestArgs/ResponseFile/ArgumentParsing/fbuild.bff"; + + FBuildForTest fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); + TEST_ASSERT( fBuild.Build( "Exe" ) ); + + // Empty + CheckParsing( "" ); + + // Multiple args + CheckParsing( "-aaa -bbb", "-aaa", "-bbb" ); + + // Quoted args + CheckParsing( R"(-aaa "-DARG")", "-aaa", "-DARG" ); + CheckParsing( R"(-aaa -D"ARG")", "-aaa", "-DARG" ); + + // Escaped quotes + CheckParsing( R"(-aaa -D\"ARG\")", "-aaa", R"(-D"ARG")" ); + CheckParsing( R"(-aaa "-D\"ARG\"")", "-aaa", R"(-D"ARG")" ); + + // Spaces inside quotes + CheckParsing( R"("-DTHING=\" \"")", R"(-DTHING=" ")" ); +} + +// CheckParsing +//------------------------------------------------------------------------------ +void TestArgs::CheckParsing( const char * commandLine, + const char * arg1, + const char * arg2 ) const +{ + // Invoke previously built exe which echoes back commands + Process p; + TEST_ASSERT( p.Spawn( "../tmp/Test/Args/ResponseFile/ArgumentParsing/Exe.exe", + commandLine, + nullptr, // working dir + nullptr ) ); // environment + + // Capture the output + AStackString<> out; + AStackString<> err; + TEST_ASSERT( p.ReadAllData( out, err ) ); + TEST_ASSERT( p.WaitForExit() == 0 ); + + // Normalize windows line endings + #if defined( __WINDOWS__ ) + out.Replace( "\r\n", "\n" ); + #endif + + // Generate expected results + AStackString<> expected; + const size_t numArgsExpected = static_cast( arg1 ? 1 : 0 ) + + static_cast( arg2 ? 1 : 0 ); + expected.Format( "%zu\n", numArgsExpected ); + if ( arg1 ) + { + expected.AppendFormat( "%s\n", arg1 ); + } + if ( arg2 ) + { + expected.AppendFormat( "%s\n", arg2 ); + } + + // Compare echoed result with what we expect + TEST_ASSERT( out == expected ); +} + +//------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp index 16004de70..199ac931d 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestBFFParsing.cpp @@ -285,7 +285,7 @@ void TestBFFParsing::ExistsDirective() const //------------------------------------------------------------------------------ void TestBFFParsing::IncludeDirective() const { - // Invalid incldue directives + // Invalid include directives TEST_PARSE_FAIL( "#include", "Error #1031" ); TEST_PARSE_FAIL( "#include BLAH", "Error #1031" ); TEST_PARSE_FAIL( "#once\n#include \"test.bff\" X", "Error #1045 - Extraneous token(s)" ); @@ -1052,7 +1052,7 @@ void TestBFFParsing::FunctionHeaders() const " .Source = 'src2'\n" " .Dest = 'dst2'\n" "}\n", "#1100 - Copy() - Target 'X' already defined." ); - // - Unexpexted function header + // - Unexpected function header TEST_PARSE_FAIL( "Settings( 'Bad' )\n" "{}", "#1021 - Settings() - Unexpected Function header." ); TEST_PARSE_FAIL( ".Name= 'X'\n" diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestBuildFBuild.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestBuildFBuild.cpp index 0db8fe20e..b01875c05 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestBuildFBuild.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestBuildFBuild.cpp @@ -85,7 +85,7 @@ FBuildStats TestBuildFBuild::BuildInternal( FBuildTestOptions options, bool useD void TestBuildFBuild::BuildClean() const { // delete files from previous runs - Array< AString > files( 1024, true ); + Array< AString > files( 1024 ); FileIO::GetFiles( AStackString<>( "../tmp/Test/BuildFBuild" ), AStackString<>( "*" ), true, &files ); for ( const AString & file : files ) { diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCLR.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCLR.cpp index 033863162..04653886e 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCLR.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCLR.cpp @@ -114,7 +114,7 @@ void TestCLR::Test() const EnsureFileDoesNotExist( "../tmp/Test/CLR/clr.lib" ); - FBuildStats stats = Build( options, false, "CLR-Target" ); // dont' use DB + FBuildStats stats = Build( options, false, "CLR-Target" ); // don't use DB EnsureFileExists( "../tmp/Test/CLR/clr.lib" ); @@ -157,7 +157,7 @@ void TestCLR::TestCache() const EnsureFileDoesNotExist( "../tmp/Test/CLR/clr.lib" ); - FBuildStats stats = Build( options, false, "CLR-Target" ); // dont' use DB + FBuildStats stats = Build( options, false, "CLR-Target" ); // don't use DB EnsureFileExists( "../tmp/Test/CLR/clr.lib" ); @@ -182,7 +182,7 @@ void TestCLR::TestParallelBuild() const EnsureFileDoesNotExist( "../tmp/Test/CLR/clrmulti.lib" ); - FBuildStats stats = Build( options, false, "CLR-Parallel-Target" ); // dont' use DB + FBuildStats stats = Build( options, false, "CLR-Parallel-Target" ); // don't use DB EnsureFileExists( "../tmp/Test/CLR/clrmulti.lib" ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp index 116f82b37..b499ba1e8 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCache.cpp @@ -39,6 +39,7 @@ class TestCache : public FBuildTest void LightCache_ImportDirective() const; void LightCache_ForceInclude() const; void LightCache_SourceDependencies() const; + void LightCache_ResponseFile() const; // MSVC Static Analysis tests const char* const mAnalyzeMSVCBFFPath = "Tools/FBuild/FBuildTest/Data/TestCache/Analyze_MSVC/fbuild.bff"; @@ -84,6 +85,7 @@ REGISTER_TESTS_BEGIN( TestCache ) REGISTER_TEST( LightCache_ImportDirective ) REGISTER_TEST( LightCache_ForceInclude ) REGISTER_TEST( LightCache_SourceDependencies ) + REGISTER_TEST( LightCache_ResponseFile ) REGISTER_TEST( Analyze_MSVC_WarningsOnly_Write ) REGISTER_TEST( Analyze_MSVC_WarningsOnly_Read ) @@ -881,6 +883,35 @@ void TestCache::LightCache_SourceDependencies() const TEST_ASSERT( GetRecordedOutput().Find( "LightCache is incompatible with -sourceDependencies" ) ); } +// LightCache_ResponseFile +//------------------------------------------------------------------------------ +void TestCache::LightCache_ResponseFile() const +{ + FBuildTestOptions options; + options.m_UseCacheWrite = true; + options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestCache/LightCache_ResponseFile/fbuild.bff"; + + // Build + FBuildForTest fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); + TEST_ASSERT( fBuild.Build( "ObjectList" ) ); + + // Ensure cache we stored using the LightCache + const FBuildStats::Stats & objStats = fBuild.GetStats().GetStatsFor( Node::OBJECT_NODE ); + TEST_ASSERT( objStats.m_NumCacheStores == 1 ); + TEST_ASSERT( objStats.m_NumLightCache == 1 ); + + // Get the discovered dependencies and ensure the include was found + // This ensures that the include paths in the args are correctly processed + // even when response files are in use + StackArray nodes; + fBuild.GetNodesOfType( Node::OBJECT_NODE, nodes ); + TEST_ASSERT( nodes.GetSize() == 1 ); + const Dependencies & deps = nodes[ 0 ]->GetDynamicDependencies(); + TEST_ASSERT( deps.GetSize() == 2 ); // main cpp plus include + TEST_ASSERT( deps[ 1 ].GetNode()->GetName().EndsWith( "include.h") ); +} + // Analyze_MSVC_WarningsOnly_Write //------------------------------------------------------------------------------ void TestCache::Analyze_MSVC_WarningsOnly_Write() const diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCachePlugin.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCachePlugin.cpp index c5b85e2d7..d62dc5dc1 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCachePlugin.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCachePlugin.cpp @@ -132,7 +132,7 @@ void TestCachePlugin::PluginOptionsSavedToDB() const FBuild f( options ); TEST_ASSERT( f.Initialize() ); - // sotre a copy of the cache params + // store a copy of the cache params cachePath = f.GetSettings()->GetCachePath(); cachePluginDLL = f.GetSettings()->GetCachePluginDLL(); TEST_ASSERT( !cachePath.IsEmpty() ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCompilationDatabase.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCompilationDatabase.cpp index 0c829631f..5fa7479a4 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCompilationDatabase.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCompilationDatabase.cpp @@ -15,6 +15,13 @@ #include "Core/FileIO/PathUtils.h" #include "Core/Strings/AStackString.h" +// Defines +//------------------------------------------------------------------------------ +#define COMPDB_COMMON_ARGS "\"-c\", " \ + "\"-Ipath with spaces\", " \ + "\"-DSTRING_DEFINE=\\\"foobar\\\"\", " \ + "\"-DSTRING_DEFINE_SPACES=\\\"foo bar\\\"\"," + // TestCompilationDatabase //------------------------------------------------------------------------------ class TestCompilationDatabase : public FBuildTest @@ -37,22 +44,12 @@ class TestCompilationDatabase : public FBuildTest //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestCompilationDatabase ) REGISTER_TEST( JSONEscape ) - REGISTER_TEST( Unquote ) REGISTER_TEST( TestObjectListInputFile ) REGISTER_TEST( TestObjectListInputPath ) REGISTER_TEST( TestUnityInputFile ) REGISTER_TEST( TestUnityInputPath ) REGISTER_TESTS_END -// TestCompilationDatabase -//------------------------------------------------------------------------------ -class CompilationDatabaseTestWrapper : public CompilationDatabase -{ -public: - static void JSONEscape( AString & string ) { JSON::Escape( string ); } - static void Unquote( AString & string ) { CompilationDatabase::Unquote( string ); } -}; - // JSONEscape //------------------------------------------------------------------------------ void TestCompilationDatabase::JSONEscape() const @@ -60,7 +57,7 @@ void TestCompilationDatabase::JSONEscape() const #define CHECK_JSONESCAPE( str, result ) \ { \ AStackString<> string( str ); \ - CompilationDatabaseTestWrapper::JSONEscape( string ); \ + JSON::Escape( string ); \ TEST_ASSERT( string == result ); \ } @@ -74,32 +71,6 @@ void TestCompilationDatabase::JSONEscape() const #undef CHECK_JSONESCAPE } -// Unquote -//------------------------------------------------------------------------------ -void TestCompilationDatabase::Unquote() const -{ - #define CHECK_UNQUOTE( str, result ) \ - { \ - AStackString<> string( str ); \ - CompilationDatabaseTestWrapper::Unquote( string ); \ - TEST_ASSERT( string == result ); \ - } - - CHECK_UNQUOTE( "", "" ) - CHECK_UNQUOTE( "\"\"", "" ) - CHECK_UNQUOTE( "''", "" ) - CHECK_UNQUOTE( "\"foo\"", "foo" ) - CHECK_UNQUOTE( "'foo'", "foo" ) - CHECK_UNQUOTE( "f\"o\"o", "foo" ) - CHECK_UNQUOTE( "f'o'o", "foo" ) - CHECK_UNQUOTE( "\"''\"", "''" ) - CHECK_UNQUOTE( "'\"\"'", "\"\"" ) - CHECK_UNQUOTE( "\"foo\"_\"bar\"", "foo_bar" ) - CHECK_UNQUOTE( "'foo'_'bar'", "foo_bar" ) - - #undef CHECK_UNQUOTE -} - // TestObjectListInputFile //------------------------------------------------------------------------------ void TestCompilationDatabase::TestObjectListInputFile() const @@ -110,7 +81,7 @@ void TestCompilationDatabase::TestObjectListInputFile() const " \"directory\": \"{WORKDIR}\",\n" " \"file\": \"{TESTDIR}file.cpp\",\n" " \"output\": \"{OUTDIR}file.result\",\n" - " \"arguments\": [\"{TESTDIR}clang.exe\", \"-c\", \"-Ipath with spaces\", \"-DSTRING_DEFINE=\\\"foobar\\\"\", \"{TESTDIR}file.cpp\", \"-o\", \"{OUTDIR}file.result\"]\n" + " \"arguments\": [\"{TESTDIR}clang.exe\", " COMPDB_COMMON_ARGS " \"{TESTDIR}file.cpp\", \"-o\", \"{OUTDIR}file.result\"]\n" " }\n" "]\n" ); @@ -126,7 +97,7 @@ void TestCompilationDatabase::TestObjectListInputPath() const " \"directory\": \"{WORKDIR}\",\n" " \"file\": \"{TESTDIR}dir{SLASH}subdir{SLASH}file.cpp\",\n" " \"output\": \"{OUTDIR}subdir{SLASH}file.result\",\n" - " \"arguments\": [\"{TESTDIR}clang.exe\", \"-c\", \"-Ipath with spaces\", \"-DSTRING_DEFINE=\\\"foobar\\\"\", \"{TESTDIR}dir{SLASH}subdir{SLASH}file.cpp\", \"-o\", \"{OUTDIR}subdir{SLASH}file.result\"]\n" + " \"arguments\": [\"{TESTDIR}clang.exe\", " COMPDB_COMMON_ARGS " \"{TESTDIR}dir{SLASH}subdir{SLASH}file.cpp\", \"-o\", \"{OUTDIR}subdir{SLASH}file.result\"]\n" " }\n" "]\n" ); @@ -142,7 +113,7 @@ void TestCompilationDatabase::TestUnityInputFile() const " \"directory\": \"{WORKDIR}\",\n" " \"file\": \"{TESTDIR}file.cpp\",\n" " \"output\": \"{OUTDIR}file.result\",\n" - " \"arguments\": [\"{TESTDIR}clang.exe\", \"-c\", \"-Ipath with spaces\", \"-DSTRING_DEFINE=\\\"foobar\\\"\", \"{TESTDIR}file.cpp\", \"-o\", \"{OUTDIR}file.result\"]\n" + " \"arguments\": [\"{TESTDIR}clang.exe\", " COMPDB_COMMON_ARGS " \"{TESTDIR}file.cpp\", \"-o\", \"{OUTDIR}file.result\"]\n" " }\n" "]\n" ); @@ -158,7 +129,7 @@ void TestCompilationDatabase::TestUnityInputPath() const " \"directory\": \"{WORKDIR}\",\n" " \"file\": \"{TESTDIR}dir{SLASH}subdir{SLASH}file.cpp\",\n" " \"output\": \"{OUTDIR}subdir{SLASH}file.result\",\n" - " \"arguments\": [\"{TESTDIR}clang.exe\", \"-c\", \"-Ipath with spaces\", \"-DSTRING_DEFINE=\\\"foobar\\\"\", \"{TESTDIR}dir{SLASH}subdir{SLASH}file.cpp\", \"-o\", \"{OUTDIR}subdir{SLASH}file.result\"]\n" + " \"arguments\": [\"{TESTDIR}clang.exe\", " COMPDB_COMMON_ARGS " \"{TESTDIR}dir{SLASH}subdir{SLASH}file.cpp\", \"-o\", \"{OUTDIR}subdir{SLASH}file.result\"]\n" " }\n" "]\n" ); @@ -207,10 +178,10 @@ void TestCompilationDatabase::DoTest( const char * bffFile, const char * target, AStackString<> slash; slash = NATIVE_SLASH_STR; - CompilationDatabaseTestWrapper::JSONEscape( workDir ); - CompilationDatabaseTestWrapper::JSONEscape( testDir ); - CompilationDatabaseTestWrapper::JSONEscape( outDir ); - CompilationDatabaseTestWrapper::JSONEscape( slash ); + JSON::Escape( workDir ); + JSON::Escape( testDir ); + JSON::Escape( outDir ); + JSON::Escape( slash ); result.Replace( "{WORKDIR}", workDir.Get() ); result.Replace( "{TESTDIR}", testDir.Get() ); @@ -219,3 +190,4 @@ void TestCompilationDatabase::DoTest( const char * bffFile, const char * target, } //------------------------------------------------------------------------------ +#undef COMPDB_COMMON_ARGS diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCompiler.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCompiler.cpp index 950865aa5..527353871 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCompiler.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCompiler.cpp @@ -62,7 +62,7 @@ void TestCompiler::BuildCompiler_Explicit() const FBuildForTest fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); - // Build a file genereated by a Compiler that we compiled + // Build a file generated by a Compiler that we compiled TEST_ASSERT( fBuild.Build( "Compiler" ) ); // Save DB for use by NoRebuild test @@ -83,7 +83,7 @@ void TestCompiler::BuildCompiler_Explicit() const FBuildForTest fBuild( options ); TEST_ASSERT( fBuild.Initialize( "../tmp/Test/TestCompiler/Explicit/explicit.fdb" ) ); - // Build a file genereated by a Compiler that we compiled + // Build a file generated by a Compiler that we compiled TEST_ASSERT( fBuild.Build( "Compiler" ) ); // Check stats @@ -102,7 +102,7 @@ void TestCompiler::BuildCompiler_Explicit() const FBuildForTest fBuild( options ); TEST_ASSERT( fBuild.Initialize( "../tmp/Test/TestCompiler/Explicit/explicit.fdb" ) ); - // Build a file genereated by a Compiler that we compiled + // Build a file generated by a Compiler that we compiled TEST_ASSERT( fBuild.Build( "Compiler" ) ); // Check stats @@ -135,7 +135,7 @@ void TestCompiler::BuildCompiler_Implicit() const FBuildForTest fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); - // Build a file genereated by a Compiler that we compiled + // Build a file generated by a Compiler that we compiled TEST_ASSERT( fBuild.Build( "ObjectList" ) ); // Save DB for use by next test @@ -188,7 +188,7 @@ void TestCompiler::BuildCompiler_Implicit() const FBuildForTest fBuild( options ); TEST_ASSERT( fBuild.Initialize( "../tmp/Test/TestCompiler/Implicit/implicit.fdb" ) ); - // Build a file genereated by a Compiler that we compiled + // Build a file generated by a Compiler that we compiled TEST_ASSERT( fBuild.Build( compilerNodeName ) ); // Save DB for use by NoRebuild test @@ -210,7 +210,7 @@ void TestCompiler::BuildCompiler_Implicit() const FBuildForTest fBuild( options ); TEST_ASSERT( fBuild.Initialize( "../tmp/Test/TestCompiler/Implicit/implicit.fdb" ) ); - // Build a file genereated by a Compiler that we compiled + // Build a file generated by a Compiler that we compiled TEST_ASSERT( fBuild.Build( compilerNodeName ) ); // Save DB for use by NoRebuild test @@ -271,7 +271,7 @@ void TestCompiler::CompilerExecutableAsDependency() const FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); - // Build a file genereated by a Compiler that we compiled + // Build a file generated by a Compiler that we compiled TEST_ASSERT( fBuild.Build( "ObjectList" ) ); // Save DB for use by NoRebuild test @@ -291,7 +291,7 @@ void TestCompiler::CompilerExecutableAsDependency_NoRebuild() const FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize( "../tmp/Test/TestCompiler/executableasdependency.fdb" ) ); - // Build a file genereated by a Compiler that we compiled + // Build a file generated by a Compiler that we compiled TEST_ASSERT( fBuild.Build( "ObjectList" ) ); // Check stats diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp index 77cb23277..9b1d30b71 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCompressor.cpp @@ -100,14 +100,8 @@ void TestCompressor::CompressSimpleHelper( const char * data, // decompress Compressor d; - if ( useZstd ) - { - TEST_ASSERT( d.DecompressZstd( compressedMem ) ); - } - else - { - TEST_ASSERT( d.Decompress( compressedMem ) ); - } + TEST_ASSERT( d.Decompress( compressedMem ) ); + const size_t decompressedSize = d.GetResultSize(); TEST_ASSERT( decompressedSize == size ); TEST_ASSERT( memcmp( data, d.GetResult(), size ) == 0 ); @@ -133,7 +127,7 @@ void TestCompressor::CompressObjFile() const void TestCompressor::CompressHelper( const char * fileName ) const { // read some test data into a file - UniquePtr< void > data; + UniquePtr< void, FreeDeletor > data; size_t dataSize; { FileStream fs; @@ -169,7 +163,7 @@ void TestCompressor::CompressHelper( const char * fileName ) const uint64_t compressedSize = 0; // Compression speed - UniquePtr< Compressor, DeleteDeletor > c; + UniquePtr c; for ( uint32_t i = 0; i < numRepeats; ++i ) { // Compress @@ -224,7 +218,7 @@ void TestCompressor::CompressHelper( const char * fileName ) const uint64_t compressedSize = 0; // Compression speed - UniquePtr< Compressor, DeleteDeletor > c; + UniquePtr c; for ( uint32_t i = 0; i < numRepeats; ++i ) { // Compress @@ -241,7 +235,7 @@ void TestCompressor::CompressHelper( const char * fileName ) const // Decompress const Timer t2; Compressor d; - TEST_ASSERT( d.DecompressZstd( c.Get()->GetResult() ) ); + TEST_ASSERT( d.Decompress( c.Get()->GetResult() ) ); TEST_ASSERT( d.GetResultSize() == dataSize ); decompressTimeTaken += (double)t2.GetElapsedMS(); @@ -267,7 +261,7 @@ void TestCompressor::CompressHelper( const char * fileName ) const //------------------------------------------------------------------------------ void TestCompressor::TestHeaderValidity() const { - UniquePtr< uint32_t > buffer( (uint32_t *)ALLOC( 1024 ) ); + UniquePtr< uint32_t, FreeDeletor > buffer( (uint32_t *)ALLOC( 1024 ) ); memset( buffer.Get(), 0, 1024 ); Compressor c; uint32_t * data = (uint32_t *)buffer.Get(); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestCopy.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestCopy.cpp index f4785362f..ff2bf31cc 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestCopy.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestCopy.cpp @@ -557,7 +557,7 @@ void TestCopy::MissingTrailingSlash() const // Parsing should fail TEST_ASSERT( fBuild.Initialize() == false ); - // Ensure the explcit error for this case is reported (not a generic one about + // Ensure the explicit error for this case is reported (not a generic one about // the target already being defined) TEST_ASSERT( GetRecordedOutput().Find( "FASTBuild Error #1400" ) ); } diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestDLL.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestDLL.cpp index 653304cb5..bc0e34a75 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestDLL.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestDLL.cpp @@ -70,7 +70,7 @@ void TestDLL::TestSingleDLL() const TEST_ASSERT( fBuild.Build( dll ) ); TEST_ASSERT( fBuild.SaveDependencyGraph( GetSingleDLLDBFileName() ) ); - // make sure all output files are as expecter + // make sure all output files are as expected EnsureFileExists( dll ); // Check stats @@ -122,7 +122,7 @@ void TestDLL::TestTwoDLLs() const TEST_ASSERT( fBuild.Build( dllB ) ); TEST_ASSERT( fBuild.SaveDependencyGraph( GetTwoDLLsDBFileName() ) ); - // make sure all output files are as expecter + // make sure all output files are as expected EnsureFileExists( dllA ); EnsureFileExists( dllB ); @@ -220,7 +220,7 @@ void TestDLL::TestDLLWithPCH() const TEST_ASSERT( fBuild.Build( dllPCH ) ); TEST_ASSERT( fBuild.SaveDependencyGraph( GetDLLWithPCHDBFileName() ) ); - // make sure all output files are as expecter + // make sure all output files are as expected EnsureFileExists( dllPCH ); // Check stats @@ -272,7 +272,7 @@ void TestDLL::TestExeWithDLL() const TEST_ASSERT( fBuild.Build( exe ) ); TEST_ASSERT( fBuild.SaveDependencyGraph( GetExeWithDLLDBFileName() ) ); - // make sure all output files are as expecter + // make sure all output files are as expected EnsureFileExists( exe ); // Check stats diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestDirectoryList.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestDirectoryList.cpp new file mode 100644 index 000000000..3d8ff315e --- /dev/null +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestDirectoryList.cpp @@ -0,0 +1,219 @@ +// TestDirectoryList.cpp +//------------------------------------------------------------------------------ + +// Includes +//------------------------------------------------------------------------------ +#include "FBuildTest.h" + +// FBuildCore +#include "Tools/FBuild/FBuildCore/Graph/DirectoryListNode.h" + +// Core +#include "Core/Containers/Array.h" +#include "Core/FileIO/PathUtils.h" +#include "Core/Strings/AStackString.h" + +// TestGraph +//------------------------------------------------------------------------------ +class TestDirectoryList : public FBuildTest +{ +private: + DECLARE_TESTS + + void Build() const; + void Names() const; +}; + +// Register Tests +//------------------------------------------------------------------------------ +REGISTER_TESTS_BEGIN( TestDirectoryList ) + REGISTER_TEST( Build ) + REGISTER_TEST( Names ) +REGISTER_TESTS_END + +// Build +//------------------------------------------------------------------------------ +void TestDirectoryList::Build() const +{ + FBuild fb; + NodeGraph ng; + + // Generate a valid DirectoryListNode name + AStackString<> name; + #if defined( __WINDOWS__ ) + const AStackString<> testFolder( "Tools\\FBuild\\FBuildTest\\Data\\TestGraph\\" ); + #else + const AStackString<> testFolder( "Tools/FBuild/FBuildTest/Data/TestGraph/" ); + #endif + Array< AString > patterns; + patterns.EmplaceBack( "library.*" ); + DirectoryListNode::FormatName( testFolder, + &patterns, + true, // recursive + false, // Don't include read-only status in hash + false, // Don't include directories + Array< AString >(), // excludePaths, + Array< AString >(), // excludeFiles, + Array< AString >(), // excludePatterns, + name ); + + // create the node, and make sure we can access it by name + DirectoryListNode * node = ng.CreateNode( name ); + node->m_Path = testFolder; + node->m_Patterns = patterns; + const BFFToken * token = nullptr; + TEST_ASSERT( node->Initialize( ng, token, nullptr ) ); + TEST_ASSERT( ng.FindNode( name ) == node ); + + TEST_ASSERT( fb.Build( node ) ); + + // make sure we got the expected results + TEST_ASSERT( node->GetFiles().GetSize() == 2 ); + #if defined( __WINDOWS__ ) + const char * fileName1 = "Tools\\FBuild\\FBuildTest\\Data\\TestGraph\\library.cpp"; + const char * fileName2 = "Tools\\FBuild\\FBuildTest\\Data\\TestGraph\\library.o"; + #else + const char * fileName1 = "Data/TestGraph/library.cpp"; + const char * fileName2 = "Data/TestGraph/library.o"; + #endif + + // returned order depends on file system + if ( node->GetFiles()[ 0 ].m_Name.EndsWith( fileName1 ) ) + { + TEST_ASSERT( node->GetFiles()[ 1 ].m_Name.EndsWith( fileName2 ) ); + } + else + { + TEST_ASSERT( node->GetFiles()[ 0 ].m_Name.EndsWith( fileName2 ) ); + TEST_ASSERT( node->GetFiles()[ 1 ].m_Name.EndsWith( fileName1 ) ); + } +} + +// Names +//------------------------------------------------------------------------------ +void TestDirectoryList::Names() const +{ + // Generate various directory list node names and ensure they are unique + // given differing search params + + // Use the same path in each case + AStackString<> path( "Tools/FBuild/FBuildTest/Data/TestDirectoryList/" ); + PathUtils::FixupFolderPath( path ); + + // Build a list of node names we can verify for uniqueness later + StackArray names; + + // All files with no special options + DirectoryListNode::FormatName( path, + nullptr, // patterns (.*) + false, // not recursive + false, // don't include read only status + false, // don't include dirs + Array(), // exclude paths + Array(), // exclude files + Array(), // exclude patterns + names.EmplaceBack() ); // result + + // Patterns + { + StackArray patterns; + patterns.EmplaceBack( ".cpp" ); + DirectoryListNode::FormatName( path, + &patterns, // patterns + false, + false, + false, + Array(), + Array(), + Array(), + names.EmplaceBack() ); + } + + // Recursive + DirectoryListNode::FormatName( path, + nullptr, + true, // recursive + false, + false, + Array(), + Array(), + Array(), + names.EmplaceBack() ); + + // Include RO status + DirectoryListNode::FormatName( path, + nullptr, + false, + true, // include read only status + false, + Array(), + Array(), + Array(), + names.EmplaceBack() ); + + // Include dirs + DirectoryListNode::FormatName( path, + nullptr, + false, + false, + true, // include dirs + Array(), + Array(), + Array(), + names.EmplaceBack() ); + + // Exclude Paths + { + StackArray excludePaths; + excludePaths.EmplaceBack( "path/path/" ); + PathUtils::FixupFolderPath( excludePaths[ 0 ] ); + DirectoryListNode::FormatName( path, + nullptr, + false, + false, + false, + excludePaths, // exclude paths + Array(), + Array(), + names.EmplaceBack() ); + } + + // Exclude Files + { + StackArray excludeFiles; + excludeFiles.EmplaceBack( "file.cpp" ); + DirectoryListNode::FormatName( path, + nullptr, + false, + false, + false, + Array(), + excludeFiles, // exclude files + Array(), + names.EmplaceBack() ); + } + + // Exclude Patterns + { + StackArray excludePatterns; + excludePatterns.EmplaceBack( "*.cpp" ); + DirectoryListNode::FormatName( path, + nullptr, + false, + false, + false, + Array(), + Array(), + excludePatterns, // exclude patterns + names.EmplaceBack() ); + } + + // Ensure all generated names were unique + names.Sort(); + for ( size_t i = 1; i < names.GetSize(); ++i ) + { + TEST_ASSERT( names[ i ] != names[ i - 1 ] ); + } +} + +//------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp index 329269a8b..ed9aebc72 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestDistributed.cpp @@ -92,8 +92,6 @@ void TestDistributed::TestHelper( const char * target, uint32_t numRemoteWorkers options.m_NumWorkerThreads = 1; options.m_NoLocalConsumptionOfRemoteJobs = true; // ensure all jobs happen on the remote worker options.m_AllowLocalRace = allowRace; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -188,9 +186,7 @@ void TestDistributed::RemoteRaceWinRemote() options.m_AllowDistributed = true; options.m_NumWorkerThreads = 1; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well options.m_NoLocalConsumptionOfRemoteJobs = true; - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -216,9 +212,7 @@ void TestDistributed::RemoteRaceSystemFailure() options.m_AllowDistributed = true; options.m_NumWorkerThreads = 1; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well options.m_NoLocalConsumptionOfRemoteJobs = true; - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; options.m_DistVerbose = true; FBuild fBuild( options ); @@ -270,8 +264,6 @@ void TestDistributed::ErrorsAreCorrectlyReported_MSVC() const options.m_NoLocalConsumptionOfRemoteJobs = true; // ensure all jobs happen on the remote worker options.m_AllowLocalRace = false; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -298,8 +290,6 @@ void TestDistributed::ErrorsAreCorrectlyReported_Clang() const options.m_NoLocalConsumptionOfRemoteJobs = true; // ensure all jobs happen on the remote worker options.m_AllowLocalRace = false; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -326,8 +316,6 @@ void TestDistributed::WarningsAreCorrectlyReported_MSVC() const options.m_NoLocalConsumptionOfRemoteJobs = true; // ensure all jobs happen on the remote worker options.m_AllowLocalRace = false; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -354,8 +342,6 @@ void TestDistributed::WarningsAreCorrectlyReported_Clang() const options.m_NoLocalConsumptionOfRemoteJobs = true; // ensure all jobs happen on the remote worker options.m_AllowLocalRace = false; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -385,8 +371,6 @@ void TestDistributed::ShutdownMemoryLeak() const options.m_NoLocalConsumptionOfRemoteJobs = true; // ensure all jobs happen on the remote worker options.m_AllowLocalRace = false; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; // Init FBuild fBuild( options ); @@ -394,7 +378,7 @@ void TestDistributed::ShutdownMemoryLeak() const // NOTE: No remote server created so jobs stay in m_DistributableJobs_Available queue - // Create thread that will abort build to simulate Crtl+C or other external stop + // Create thread that will abort build to simulate Ctrl+C or other external stop class Helper { public: @@ -429,7 +413,7 @@ void TestDistributed::ShutdownMemoryLeak() const // Start build and check it was aborted TEST_ASSERT( fBuild.Build( "ShutdownMemoryLeak" ) == false ); - TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Error: BUILD FAILED: ShutdownMemoryLeak" ) ); + TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Incomplete: ShutdownMemoryLeak" ) ); thread.Join(); @@ -447,8 +431,6 @@ void TestDistributed::TestZiDebugFormat() const options.m_NoLocalConsumptionOfRemoteJobs = true; // ensure all jobs happen on the remote worker options.m_AllowLocalRace = false; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -468,8 +450,6 @@ void TestDistributed::TestZiDebugFormat_Local() const options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestDistributed/fbuild.bff"; options.m_AllowDistributed = true; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -492,8 +472,6 @@ void TestDistributed::D8049_ToolLongDebugRecord() const options.m_NoLocalConsumptionOfRemoteJobs = true; // ensure all jobs happen on the remote worker options.m_AllowLocalRace = false; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -508,7 +486,7 @@ void TestDistributed::D8049_ToolLongDebugRecord() const //------------------------------------------------------------------------------ void TestDistributed::CleanMessageToPreventMSBuildFailure() const { - // Error should be indentical except for a single remove colon + // Error should be identical except for a single remove colon { const AStackString<> in( "C:\\Windows\\TEMP\\.fbuild.tmp\\0x00000000\\" "core_1018\\F436D72E\\Module.MergeActors.cpp :" diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp index 71ef3a07a..ec8fd84cc 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestExec.cpp @@ -29,6 +29,7 @@ class TestExec : public FBuildTest void Build_ExecCommand_ExpectedFailures() const; void Build_ExecEnvCommand() const; void Exclusions() const; + void PreBuildDependencies() const; }; // Register Tests @@ -43,6 +44,7 @@ REGISTER_TESTS_BEGIN( TestExec ) REGISTER_TEST( Build_ExecCommand_ExpectedFailures ) REGISTER_TEST( Build_ExecEnvCommand ) REGISTER_TEST( Exclusions ) + REGISTER_TEST( PreBuildDependencies ) REGISTER_TESTS_END // Helpers @@ -305,7 +307,7 @@ void TestExec::Build_ExecCommand_ExpectedFailures() const fBuild.Initialize( "../tmp/Test/Exec/exec.fdb" ); // build - Array< AString > targets( 2, false ); + Array< AString > targets( 2 ); targets.EmplaceBack( "ExecCommandTest_OneInput_ReturnCode_ExpectFail" ); targets.EmplaceBack( "ExecCommandTest_OneInput_WrongOutput_ExpectFail" ); TEST_ASSERT( !fBuild.Build( targets ) ); @@ -370,3 +372,15 @@ void TestExec::Exclusions() const } //------------------------------------------------------------------------------ +void TestExec::PreBuildDependencies() const +{ + FBuildTestOptions options; + options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestExec/PreBuildDependencies/prebuilddependencies.bff"; + FBuildForTest fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); + + // build (via alias) + TEST_ASSERT( fBuild.Build( "Exec" ) ); +} + +//------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp index 3b7936ec7..070573f0f 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestFastCancel.cpp @@ -7,6 +7,7 @@ #include "Tools/FBuild/FBuildCore/FBuild.h" +#include "Core/Math/Conversions.h" #include "Core/Process/SystemMutex.h" #include "Core/Process/Thread.h" #include "Core/Strings/AStackString.h" @@ -18,42 +19,108 @@ class TestFastCancel : public FBuildTest private: DECLARE_TESTS + template + void BuildFailure() const; void Cancel() const; }; // Register Tests //------------------------------------------------------------------------------ REGISTER_TESTS_BEGIN( TestFastCancel ) - #if defined( __WINDOWS__ ) - // TODO:LINUX TODO:OSX - Fix and enable this test - REGISTER_TEST( Cancel ) - #endif + REGISTER_TEST( BuildFailure<0> ) + REGISTER_TEST( BuildFailure<1> ) + REGISTER_TEST( BuildFailure<2> ) + REGISTER_TEST( BuildFailure<3> ) + REGISTER_TEST( BuildFailure<4> ) + REGISTER_TEST( BuildFailure<5> ) + REGISTER_TEST( BuildFailure<6> ) + REGISTER_TEST( BuildFailure<7> ) + REGISTER_TEST( Cancel ) REGISTER_TESTS_END +//------------------------------------------------------------------------------ +template +void TestFastCancel::BuildFailure() const +{ + // Check that builds with failures are finalized correctly + // under a variety of conditions + + // Derive individual settings from the seed + const bool allowFastCancel = ( ( SEED & 0x1 ) != 0 ); // Can in-flight tasks be aborted? + const bool stopOnError = ( ( SEED & 0x2 ) != 0 ); // Stop building after encountering an error? + const bool reverseOrder = ( ( SEED & 0x4 ) != 0 ); // Reverse order of targets + + FBuildTestOptions options; + options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestFastCancel/BuildFailure/fbuild.bff"; + options.m_ShowSummary = false; // Reduce build output spam + options.m_FastCancel = allowFastCancel; + options.m_StopOnFirstError = stopOnError; + + // Flip order of targets passed on command line - results should be + // independent of this + Array targets; + if ( reverseOrder ) + { + targets.EmplaceBack( "b" ); + targets.EmplaceBack( "a" ); + } + else + { + targets.EmplaceBack( "a" ); + targets.EmplaceBack( "b" ); + } + + // Init + FBuild fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); + + // Start build and check it failed as expected + TEST_ASSERT( fBuild.Build( targets ) == false ); + + // Item 'a' should always fail (contains a compile error) + TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Error: BUILD FAILED: a" ) ); + + // Item 'b' might succeed, but could be aborted depending on settings and + // other factors + if constexpr ( stopOnError == false ) + { + // If we don't stop on errors, 'a' should always compile because it is + // unrelated to 'b' + TEST_ASSERT( GetRecordedOutput().Find( "FBuild: OK: b" ) ); + } + else + { + // For multithreaded builds, thread scheduling and other timing is + // non-deterministic, so 'b' may or may not complete, but it should + // never fail + TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Incomplete: b" ) || + GetRecordedOutput().Find( "FBuild: OK: b" ) ); + } +} + // CancelHelperThread //------------------------------------------------------------------------------ static uint32_t CancelHelperThread( void * ) { const Timer t; - // Wait for spawned processes to own all mutexes - SystemMutex mutex1( "FASTBuildFastCancelTest1" ); - SystemMutex mutex2( "FASTBuildFastCancelTest2" ); - SystemMutex mutex3( "FASTBuildFastCancelTest3" ); + // The Exec step in our test spawns a hierarchy of processes. When the leaf + // process of the tree is created, it acquires the final mutex + // "FASTBuildFastCancelTest4" + // + // If we can acquire that, the process has either not started or has failed in + // an unexpected way. SystemMutex mutex4( "FASTBuildFastCancelTest4" ); - SystemMutex * mutexes[] = { &mutex1, &mutex2, &mutex3, &mutex4 }; - for ( SystemMutex * mutex : mutexes ) + uint32_t sleepTimeMS = 1; + while ( mutex4.TryLock() == true ) { - // if we acquired the lock, the child process is not yet spawned - while ( mutex->TryLock() == true ) - { - // unlock so child can acquire - mutex->Unlock(); - - // Wait before trying again - Thread::Sleep( 10 ); - TEST_ASSERT( t.GetElapsed() < 5.0f ); // Ensure test doesn't hang if broken - } + // unlock so child can acquire + mutex4.Unlock(); + + // Wait before trying again, waiting longer each time up to a limit + Thread::Sleep( sleepTimeMS ); + sleepTimeMS = Math::Min( ( sleepTimeMS * 2 ), 128 ); + TEST_ASSERT( t.GetElapsed() < 5.0f ); // Ensure test doesn't hang if broken } // Once we are here, all child processes have acquired locks and will wait forever @@ -70,30 +137,27 @@ void TestFastCancel::Cancel() const FBuildTestOptions options; options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestFastCancel/Cancel/fbuild.bff"; options.m_ForceCleanBuild = true; - options.m_EnableMonitor = true; // make sure monitor code paths are tested as well // Init FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); - // We'll coordinate processes with these - SystemMutex mutex1( "FASTBuildFastCancelTest1" ); - SystemMutex mutex2( "FASTBuildFastCancelTest2" ); - SystemMutex mutex3( "FASTBuildFastCancelTest3" ); - SystemMutex mutex4( "FASTBuildFastCancelTest4" ); - SystemMutex * mutexes[] = { &mutex1, &mutex2, &mutex3, &mutex4 }; - // Create thread that will abort build once all processes are spawned Thread thread; thread.Start( CancelHelperThread ); // Start build and check it was aborted TEST_ASSERT( fBuild.Build( "Cancel" ) == false ); - TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Error: BUILD FAILED: Cancel" ) ); + TEST_ASSERT( GetRecordedOutput().Find( "FBuild: Incomplete: Cancel" ) ); thread.Join(); // Ensure that processes were killed + SystemMutex mutex1( "FASTBuildFastCancelTest1" ); + SystemMutex mutex2( "FASTBuildFastCancelTest2" ); + SystemMutex mutex3( "FASTBuildFastCancelTest3" ); + SystemMutex mutex4( "FASTBuildFastCancelTest4" ); + SystemMutex * mutexes[] = { &mutex1, &mutex2, &mutex3, &mutex4 }; for ( SystemMutex * mutex : mutexes ) { // We should be able to acquire each lock diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp index 2e9029f13..d69bfb7ff 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestGraph.cpp @@ -49,7 +49,6 @@ class TestGraph : public FBuildTest void TestCleanPathPartial() const; void SingleFileNode() const; void SingleFileNodeMissing() const; - void TestDirectoryListNode() const; void TestSerialization() const; void TestDeepGraph() const; void TestNoStopOnFirstError() const; @@ -71,7 +70,6 @@ REGISTER_TESTS_BEGIN( TestGraph ) REGISTER_TEST( TestCleanPathPartial ) REGISTER_TEST( SingleFileNode ) REGISTER_TEST( SingleFileNodeMissing ) - REGISTER_TEST( TestDirectoryListNode ) REGISTER_TEST( TestSerialization ) REGISTER_TEST( TestDeepGraph ) REGISTER_TEST( TestNoStopOnFirstError ) @@ -198,64 +196,6 @@ void TestGraph::SingleFileNodeMissing() const TEST_ASSERT( fb.Build( node ) == true ); } -// TestDirectoryListNode -//------------------------------------------------------------------------------ -void TestGraph::TestDirectoryListNode() const -{ - FBuild fb; - NodeGraph ng; - - // Generate a valid DirectoryListNode name - AStackString<> name; - #if defined( __WINDOWS__ ) - const AStackString<> testFolder( "Tools\\FBuild\\FBuildTest\\Data\\TestGraph\\" ); - #else - const AStackString<> testFolder( "Tools/FBuild/FBuildTest/Data/TestGraph/" ); - #endif - Array< AString > patterns; - patterns.EmplaceBack( "library.*" ); - DirectoryListNode::FormatName( testFolder, - &patterns, - true, // recursive - false, // Don't include read-only status in hash - false, // Don't include directories - Array< AString >(), // excludePaths, - Array< AString >(), // excludeFiles, - Array< AString >(), // excludePatterns, - name ); - - // create the node, and make sure we can access it by name - DirectoryListNode * node = ng.CreateNode( name ); - node->m_Path = testFolder; - node->m_Patterns = patterns; - const BFFToken * token = nullptr; - TEST_ASSERT( node->Initialize( ng, token, nullptr ) ); - TEST_ASSERT( ng.FindNode( name ) == node ); - - TEST_ASSERT( fb.Build( node ) ); - - // make sure we got the expected results - TEST_ASSERT( node->GetFiles().GetSize() == 2 ); - #if defined( __WINDOWS__ ) - const char * fileName1 = "Tools\\FBuild\\FBuildTest\\Data\\TestGraph\\library.cpp"; - const char * fileName2 = "Tools\\FBuild\\FBuildTest\\Data\\TestGraph\\library.o"; - #else - const char * fileName1 = "Data/TestGraph/library.cpp"; - const char * fileName2 = "Data/TestGraph/library.o"; - #endif - - // returned order depends on file system - if ( node->GetFiles()[ 0 ].m_Name.EndsWith( fileName1 ) ) - { - TEST_ASSERT( node->GetFiles()[ 1 ].m_Name.EndsWith( fileName2 ) ); - } - else - { - TEST_ASSERT( node->GetFiles()[ 0 ].m_Name.EndsWith( fileName2 ) ); - TEST_ASSERT( node->GetFiles()[ 1 ].m_Name.EndsWith( fileName1 ) ); - } -} - // TestSerialization //------------------------------------------------------------------------------ void TestGraph::TestSerialization() const @@ -293,8 +233,8 @@ void TestGraph::TestSerialization() const fs1.Open( dbFile1 ); fs2.Open( dbFile2 ); TEST_ASSERT( fs1.GetFileSize() == fs2.GetFileSize() ); // size should be the same - UniquePtr< char > buffer1( (char *)ALLOC( MEGABYTE ) ); - UniquePtr< char > buffer2( (char *)ALLOC( MEGABYTE ) ); + UniquePtr< char, FreeDeletor > buffer1( (char *)ALLOC( MEGABYTE ) ); + UniquePtr< char, FreeDeletor > buffer2( (char *)ALLOC( MEGABYTE ) ); uint32_t remaining = (uint32_t)fs1.GetFileSize(); while ( remaining > 0 ) { @@ -616,7 +556,7 @@ void TestGraph::DBLocationChanged() const TEST_ASSERT( FileIO::FileCopy( dbFile1, dbFile2 ) ); } - // Moving a DB should result in a messsage and a failed build + // Moving a DB should result in a message and a failed build { FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize( dbFile2 ) == false ); @@ -761,7 +701,7 @@ void TestGraph::BFFDirtied() const TEST_ASSERT( t.GetElapsed() < 10.0f ); // Sanity check fail test after a longtime } - // Modity BFF (make it empty) + // Modify BFF (make it empty) { FileStream fs; TEST_ASSERT( fs.Open( copyOfBFF, FileStream::WRITE_ONLY ) ); @@ -898,7 +838,7 @@ void TestGraph::FixupErrorPaths() const //------------------------------------------------------------------------------ void TestGraph::CyclicDependency() const { - // Statically defined cyclice dependencies are detected at BFF parse time, + // Statically defined cyclic dependencies are detected at BFF parse time, // but additional ones can be created at build time, so have to be detected // at build time. // diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp index 3033cf3d4..ac0dc8bfd 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestIf.cpp @@ -144,7 +144,7 @@ void TestIf::IfFunctionBool() const TEST_EXP_TRUE( ".Bool = true", "false != .Bool" ); TEST_EXP_FALSE( ".Bool = true", ".Bool != .Bool" ); - // Compound Exps + // Compound expressions TEST_EXP_TRUE( ".True = true\n .False = false", ".True && true || .False" ); TEST_EXP_TRUE( ".True = true\n .False = false", "true && .True && true || !false || .False" ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestLinker.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestLinker.cpp index c0ec11396..637ab87d9 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestLinker.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestLinker.cpp @@ -251,7 +251,7 @@ void TestLinker::IncrementalLinking_MSVC() const { FBuildTestOptions options; options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestLinker/IncrementalLinking_MSVC/fbuild.bff"; - options.m_ShowCommandOutput = true; // Show linker output so we can check analyze /VERBOSE outptut + options.m_ShowCommandOutput = true; // Show linker output so we can check analyze /VERBOSE output // Files const char * dbFile = "../tmp/Test/TestLinker/IncrementalLinking_MSVC/fbuild.fdb"; @@ -322,7 +322,7 @@ void TestLinker::IncrementalLinking_MSVC() const const AStackString<> output( GetRecordedOutput().Get() + sizeOfRecordedOutput ); - // Should see incremental linking messsages.. + // Should see incremental linking messages.. TEST_ASSERT( output.Find( "modules have changed since prior linking" ) ); // .. but should not be a full link diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp index 0da2ccf61..51bf06f28 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestNodeReflection.cpp @@ -159,7 +159,7 @@ REGISTER_TESTS_BEGIN( TestNodeReflection ) REGISTER_TEST( MetaPath_ArrayOfStrings_Required_EmptyElement ) REGISTER_TESTS_END -// Helper classese +// Helper classes //------------------------------------------------------------------------------ // BaseNode diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestObject.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestObject.cpp index e04de44e3..f2b75fd8b 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestObject.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestObject.cpp @@ -399,7 +399,7 @@ void TestObject::CacheUsingRelativePaths() const TEST_ASSERT( f.Open( objFileA ) ); buffer.SetLength( (uint32_t)f.GetFileSize() ); TEST_ASSERT( f.ReadBuffer( buffer.Get(), f.GetFileSize() ) == f.GetFileSize() ); - buffer.Replace( (char)0, ' ' ); // Make string seaches simpler + buffer.Replace( (char)0, ' ' ); // Make string searches simpler } // Check __FILE__ paths are relative @@ -484,7 +484,7 @@ void TestObject::SourceMapping() const TEST_ASSERT( f.Open( objFile ) ); buffer.SetLength( (uint32_t)f.GetFileSize() ); TEST_ASSERT( f.ReadBuffer( buffer.Get(), f.GetFileSize() ) == f.GetFileSize() ); - buffer.Replace( (char)0, ' ' ); // Make string seaches simpler + buffer.Replace( (char)0, ' ' ); // Make string searches simpler } TEST_ASSERT( buffer.Find( "/fastbuild-test-mapping" ) ); @@ -521,7 +521,6 @@ void TestObject::ClangExplicitLanguageType() const options.m_AllowDistributed = true; options.m_NoLocalConsumptionOfRemoteJobs = true; options.m_AllowLocalRace = false; - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -539,7 +538,7 @@ void TestObject::ClangExplicitLanguageType() const //------------------------------------------------------------------------------ void TestObject::ClangDependencyArgs() const { - // Ensure explicitly depedency options are removed from the second pass of + // Ensure explicitly dependency options are removed from the second pass of // compilation. Some integrations (like Unreal) use these commands and process // the output. const char* const configFile = "Tools/FBuild/FBuildTest/Data/TestObject/ClangDependencyArgs/fbuild.bff"; @@ -566,7 +565,6 @@ void TestObject::ClangDependencyArgs() const options.m_AllowDistributed = true; options.m_NoLocalConsumptionOfRemoteJobs = true; options.m_AllowLocalRace = false; - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -626,7 +624,6 @@ void TestObject::CLDependencyArgs() const options.m_AllowDistributed = true; options.m_NoLocalConsumptionOfRemoteJobs = true; options.m_AllowLocalRace = false; - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); @@ -660,7 +657,6 @@ void TestObject::CLDependencyArgs() const options.m_AllowDistributed = true; options.m_NoLocalConsumptionOfRemoteJobs = false; options.m_AllowLocalRace = true; - options.m_DistributionPort = Protocol::PROTOCOL_TEST_PORT; FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize() ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp index ef9760769..418bfdacf 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestObjectList.cpp @@ -145,7 +145,7 @@ void TestObjectList::ExtraOutputFolders_PathExtraction() const " /FaTools\\FBuild\\FBuildTest\\Data/../../../../../tmp/Test/ObjectList/ExtraOutputPaths/ObjectList//asm/file.asm" " /sourceDependencies Tools\\FBuild\\FBuildTest\\Data/../../../../../tmp/Test/ObjectList/ExtraOutputPaths/ObjectList//srcDeps/file.json"); - // Getthe paths + // Get the paths AStackString<> pdbPath, asmPath, sourceDependenciesPath; FunctionObjectList::GetExtraOutputPaths( args, pdbPath, asmPath, sourceDependenciesPath ); @@ -233,7 +233,7 @@ void TestObjectList::ObjectListChaining_Bad() const // // NOTE: This setup has some fundamental problems and should generally not be used. // For example, deletion of a file in the source directory won't remove the - // interemediate file and copies of the stale file can still occur, and/or + // intermediate file and copies of the stale file can still occur, and/or // cause build failures. // // The preferred mechanism is to chain ObjectLists together via .CompileInputObjectList diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestPrecompiledHeaders.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestPrecompiledHeaders.cpp index 9d825302c..6ae7affd9 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestPrecompiledHeaders.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestPrecompiledHeaders.cpp @@ -666,7 +666,7 @@ void TestPrecompiledHeaders::PrecompiledHeaderCacheAnalyze_MSVC() const } } -// TestPCHClang +// TestPCHClangWindows //------------------------------------------------------------------------------ #if defined( __WINDOWS__ ) void TestPrecompiledHeaders::TestPCHClangWindows() const @@ -678,8 +678,10 @@ void TestPrecompiledHeaders::PrecompiledHeaderCacheAnalyze_MSVC() const AStackString<> obj( "../tmp/Test/PrecompiledHeaders/Clang-Windows/PCHUser.obj" ); AStackString<> pch( "../tmp/Test/PrecompiledHeaders/Clang-Windows/PrecompiledHeader.pch" ); + AStackString<> pchObj( "../tmp/Test/PrecompiledHeaders/Clang-Windows/PrecompiledHeader.pch.obj" ); EnsureFileDoesNotExist( obj ); EnsureFileDoesNotExist( pch ); + EnsureFileDoesNotExist( pchObj ); FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize( nullptr ) ); @@ -691,21 +693,21 @@ void TestPrecompiledHeaders::PrecompiledHeaderCacheAnalyze_MSVC() const EnsureFileExists( obj ); EnsureFileExists( pch ); + EnsureFileExists( pchObj ); // Check stats // Seen, Built, Type - CheckStatsNode ( 4, 2, Node::FILE_NODE ); // pch.cpp + pch.h + slow.h + pchuser.cpp + CheckStatsNode ( 5, 3, Node::FILE_NODE ); // pch.cpp + pch.h + slow.h + pchuser.cpp + linker CheckStatsNode ( 1, 1, Node::COMPILER_NODE ); CheckStatsNode ( 2, 2, Node::OBJECT_NODE );// obj + pch obj CheckStatsNode ( 1, 1, Node::OBJECT_LIST_NODE ); - CheckStatsTotal( 8, 6 ); // check we wrote all objects to the cache TEST_ASSERT( fBuild.GetStats().GetStatsFor( Node::OBJECT_NODE ).m_NumCacheStores == 2 ); // can store the pch & the obj using it } #endif -// TestPCHClang_NoRebuild +// TestPCHClangWindows_NoRebuild //------------------------------------------------------------------------------ #if defined( __WINDOWS__ ) void TestPrecompiledHeaders::TestPCHClangWindows_NoRebuild() const @@ -713,9 +715,6 @@ void TestPrecompiledHeaders::PrecompiledHeaderCacheAnalyze_MSVC() const FBuildTestOptions options; options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestPrecompiledHeaders/fbuild.bff"; - AStackString<> obj( "../tmp/Test/PrecompiledHeaders/Clang-Windows/PCHUser.obj" ); - AStackString<> pch( "../tmp/Test/PrecompiledHeaders/Clang-Windows/PrecompiledHeader.pch" ); - FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize( GetPCHDBClangFileName() ) ); @@ -726,15 +725,14 @@ void TestPrecompiledHeaders::PrecompiledHeaderCacheAnalyze_MSVC() const // Check stats // Seen, Built, Type - CheckStatsNode ( 4, 4, Node::FILE_NODE ); // pch.cpp + pch.h + slow.h + pchuser.cpp + CheckStatsNode ( 5, 5, Node::FILE_NODE ); // pch.cpp + pch.h + slow.h + pchuser.cpp + linker CheckStatsNode ( 1, 0, Node::COMPILER_NODE ); CheckStatsNode ( 2, 0, Node::OBJECT_NODE );// obj + pch obj CheckStatsNode ( 1, 0, Node::OBJECT_LIST_NODE ); - CheckStatsTotal( 8, 4 ); } #endif -// TestPCHClangWithCache +// TestPCHClangWindowsWithCache //------------------------------------------------------------------------------ #if defined( __WINDOWS__ ) void TestPrecompiledHeaders::TestPCHClangWindowsWithCache() const @@ -746,8 +744,10 @@ void TestPrecompiledHeaders::PrecompiledHeaderCacheAnalyze_MSVC() const AStackString<> obj( "../tmp/Test/PrecompiledHeaders/Clang-Windows/PCHUser.obj" ); AStackString<> pch( "../tmp/Test/PrecompiledHeaders/Clang-Windows/PrecompiledHeader.pch" ); + AStackString<> pchObj( "../tmp/Test/PrecompiledHeaders/Clang-Windows/PrecompiledHeader.pch.obj" ); EnsureFileDoesNotExist( obj ); EnsureFileDoesNotExist( pch ); + EnsureFileDoesNotExist( pchObj ); FBuild fBuild( options ); TEST_ASSERT( fBuild.Initialize( nullptr ) ); @@ -762,18 +762,17 @@ void TestPrecompiledHeaders::PrecompiledHeaderCacheAnalyze_MSVC() const // Check stats // Seen, Built, Type - CheckStatsNode ( 4, 2, Node::FILE_NODE ); // pch.cpp + pch.h + slow.h + pchuser.cpp + CheckStatsNode ( 5, 3, Node::FILE_NODE ); // pch.cpp + pch.h + slow.h + pchuser.cpp + linker CheckStatsNode ( 1, 1, Node::COMPILER_NODE ); CheckStatsNode ( 2, 0, Node::OBJECT_NODE );// obj + pch obj CheckStatsNode ( 1, 1, Node::OBJECT_LIST_NODE ); - CheckStatsTotal( 8, 4 ); // check we got both objects from the cache TEST_ASSERT( fBuild.GetStats().GetStatsFor( Node::OBJECT_NODE ).m_NumCacheHits == 2 ); // pch & the obj using it } #endif -// TestPCHClangWithCache_NoRebuild +// TestPCHClangWindowsWithCache_NoRebuild //------------------------------------------------------------------------------ #if defined( __WINDOWS__ ) void TestPrecompiledHeaders::TestPCHClangWindowsWithCache_NoRebuild() const @@ -791,11 +790,10 @@ void TestPrecompiledHeaders::PrecompiledHeaderCacheAnalyze_MSVC() const // Check stats // Seen, Built, Type - CheckStatsNode ( 4, 4, Node::FILE_NODE ); // pch.cpp + pch.h + slow.h + pchuser.cpp + CheckStatsNode ( 5, 5, Node::FILE_NODE ); // pch.cpp + pch.h + slow.h + pchuser.cpp + linker CheckStatsNode ( 1, 0, Node::COMPILER_NODE ); CheckStatsNode ( 2, 0, Node::OBJECT_NODE );// obj + pch obj CheckStatsNode ( 1, 0, Node::OBJECT_LIST_NODE ); - CheckStatsTotal( 8, 4 ); } #endif diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp index a86bfd574..aff45b2e5 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestProjectGeneration.cpp @@ -322,7 +322,7 @@ void TestProjectGeneration::TestFunction_Speed() const Random r( 1234567 ); // Deterministic seed const size_t numFiles = 5000; const size_t maxSubDirDepth = 8; - Array< AString > files( numFiles, false ); + Array< AString > files( numFiles ); for ( size_t i = 0; i < numFiles; ++i ) { AStackString<> fileName( baseDir ); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestResources.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestResources.cpp index 0ea7133ab..16d9184fa 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestResources.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestResources.cpp @@ -51,7 +51,7 @@ void TestResources::BuildResource() const // make sure all output files are as expected EnsureFileExists( binRes ); - // spawn exe which does a runtime check that the resource is availble + // spawn exe which does a runtime check that the resource is available Process p; TEST_ASSERT( p.Spawn( "../tmp/Test/Resources/exe.exe", nullptr, nullptr, nullptr ) ); const int ret = p.WaitForExit(); diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp index 0ffa693ff..b8fd641c7 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestUnity.cpp @@ -202,7 +202,7 @@ void TestUnity::DetectDeletedUnityFiles() const // Build FBuildTestOptions options; options.m_ShowBuildReason = true; - BuildGenerate( options, false ); // don't laod DB + BuildGenerate( options, false ); // don't load DB EnsureFileExists( "../tmp/Test/Unity/Unity1.cpp" ); EnsureFileExists( "../tmp/Test/Unity/Unity2.cpp" ); @@ -593,7 +593,7 @@ void TestUnity::LinkMultiple() const //------------------------------------------------------------------------------ void TestUnity::LinkMultiple_InputFiles() const { - // Same as LinkMultiple but with files explicitily specified instead o + // Same as LinkMultiple but with files explicitly specified instead o // discovering them via directory listings // Code files generated/used by this test @@ -767,7 +767,7 @@ void TestUnity::SortFiles() const Array< FileIO::FileInfo * > m_HelperFileInfos; }; - // Helper marcos to reduce boilerplate code + // Helper macros to reduce boilerplate code #define SORT( ... ) \ do { \ const char * const inputs[] = { __VA_ARGS__ }; \ @@ -924,7 +924,7 @@ void TestUnity::CacheUsingRelativePaths() const TEST_ASSERT( f.Open( objFileA ) ); buffer.SetLength( (uint32_t)f.GetFileSize() ); TEST_ASSERT( f.ReadBuffer( buffer.Get(), f.GetFileSize() ) == f.GetFileSize() ); - buffer.Replace( (char)0, ' ' ); // Make string seaches simpler + buffer.Replace( (char)0, ' ' ); // Make string searches simpler } // Check __FILE__ paths are relative diff --git a/Code/Tools/FBuild/FBuildTest/Tests/TestWarnings.cpp b/Code/Tools/FBuild/FBuildTest/Tests/TestWarnings.cpp index 4f5b569ef..94bce4e44 100644 --- a/Code/Tools/FBuild/FBuildTest/Tests/TestWarnings.cpp +++ b/Code/Tools/FBuild/FBuildTest/Tests/TestWarnings.cpp @@ -27,10 +27,9 @@ class TestWarnings : public FBuildTest REGISTER_TESTS_BEGIN( TestWarnings ) REGISTER_TEST( WarningsAreShown ) #if defined( __WINDOWS__ ) - REGISTER_TEST( PragmaMessageWarningsAreShown ) - #endif - #if defined( __WINDOWS__ ) || defined( __OSX__ ) + // TODDO: B Enable for other platforms (fix two-pass issues) REGISTER_TEST( ClangMacroExpansion ) + REGISTER_TEST( PragmaMessageWarningsAreShown ) #endif REGISTER_TESTS_END @@ -45,6 +44,13 @@ void TestWarnings::WarningsAreShown() const TEST_ASSERT( fBuild.Initialize() ); TEST_ASSERT( fBuild.Build( "Warnings" ) ); + + // Ensure warning was emitted + #if defined( __WINDOWS__ ) + TEST_ASSERT( GetRecordedOutput().Find( "unreferenced local variable" ) ); + #else + TEST_ASSERT( GetRecordedOutput().Find( "-Wunused-variable" ) ); + #endif } // PragmaMessageWarningsAreShown @@ -58,25 +64,25 @@ void TestWarnings::PragmaMessageWarningsAreShown() const TEST_ASSERT( fBuild.Initialize() ); TEST_ASSERT( fBuild.Build( "PragmaMessage" ) ); + + // Ensure message was emitted + TEST_ASSERT( GetRecordedOutput().Find( "Optimization force disabled" ) ); } // ClangMacroExpansion //------------------------------------------------------------------------------ void TestWarnings::ClangMacroExpansion() const { - #if defined( __WINDOWS__ ) - // TODO:A Check if this is still relevant for newer versions of Clang - // The warning this test relies on has change and this test may need to - // be updated - #else - FBuildTestOptions options; - options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestWarnings/ClangMacroExpansion/fbuild.bff"; + FBuildTestOptions options; + options.m_ConfigFile = "Tools/FBuild/FBuildTest/Data/TestWarnings/ClangMacroExpansion/fbuild.bff"; - FBuild fBuild( options ); - TEST_ASSERT( fBuild.Initialize() ); + FBuild fBuild( options ); + TEST_ASSERT( fBuild.Initialize() ); - TEST_ASSERT( fBuild.Build( "ClangMacroExpansion" ) ); - #endif + TEST_ASSERT( fBuild.Build( "ClangMacroExpansion" ) ); + + // Ensure message was emitted + TEST_ASSERT( GetRecordedOutput().Find( "-Wtautological-unsigned-zero-compare" ) ); } //------------------------------------------------------------------------------ diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/IdleDetection.cpp b/Code/Tools/FBuild/FBuildWorker/Worker/IdleDetection.cpp index cb2736ea5..2a42ef389 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/IdleDetection.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Worker/IdleDetection.cpp @@ -40,7 +40,7 @@ IdleDetection::IdleDetection() , m_IsIdleCurrent( 0.0f ) , m_IdleSmoother( 0 ) , m_IdleFloatSmoother( 0 ) - , m_ProcessesInOurHierarchy( 32, true ) + , m_ProcessesInOurHierarchy( 32 ) , m_LastTimeIdle( 0 ) , m_LastTimeBusy( 0 ) { @@ -124,8 +124,8 @@ bool IdleDetection::IsIdleInternal( uint32_t idleThresholdPercent, float & idleC m_LastTimeBusy = ( userTime + kernTime ); } - // if the total CPU time is below the idle theshold, we don't need to - // check to know acurately what the cpu use of FASTBuild is + // if the total CPU time is below the idle threshold, we don't need to + // check to know accurately what the cpu use of FASTBuild is if ( m_CPUUsageTotal < (float)idleThresholdPercent ) { idleCurrent = 1.0f; @@ -198,7 +198,7 @@ bool IdleDetection::IsIdleInternal( uint32_t idleThresholdPercent, float & idleC // First line should be system totals if ( procStat.BeginsWithI( "cpu" ) ) { - Array< uint32_t > values( 10, true ); + Array< uint32_t > values( 10 ); const char * pos = procStat.Get() + 4; // skip "cpu " for ( ;; ) { @@ -262,7 +262,7 @@ bool IdleDetection::IsIdleInternal( uint32_t idleThresholdPercent, float & idleC if ( GetProcessInfoString( AStackString<>().Format( "/proc/%u/stat", pi.m_PID ).Get(), processInfo ) ) { - Array< AString > tokens( 32, true ); + Array< AString > tokens( 32 ); processInfo.Tokenize( tokens, ' ' ); if ( tokens.GetSize() >= 15 ) { @@ -332,7 +332,7 @@ void IdleDetection::UpdateProcessList() } else { - // gracefully handle failure to open proces + // gracefully handle failure to open process // maybe it closed before we got to it } } @@ -409,7 +409,7 @@ void IdleDetection::UpdateProcessList() } // Item index 3 (0-based) is the parent PID - Array< AString > tokens( 32, true ); + Array< AString > tokens( 32 ); processInfo.Tokenize( tokens, ' ' ); const uint32_t parentPID = strtoul( tokens[ 3 ].Get(), nullptr, 10 ); diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h b/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h index cb1ae6a2f..ff05991fe 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h +++ b/Code/Tools/FBuild/FBuildWorker/Worker/WorkerSettings.h @@ -37,7 +37,7 @@ class WorkerSettings : public Singleton< WorkerSettings > // Start minimized void SetStartMinimized( bool startMinimized ); - inline bool GetStartMinimzed() const { return m_StartMinimized; } + inline bool GetStartMinimized() const { return m_StartMinimized; } // Time settings were last changed/written to disk uint64_t GetSettingsWriteTime() const { return m_SettingsWriteTime; } diff --git a/Code/Tools/FBuild/FBuildWorker/Worker/WorkerWindow.cpp b/Code/Tools/FBuild/FBuildWorker/Worker/WorkerWindow.cpp index 57200f2b7..215a968cf 100644 --- a/Code/Tools/FBuild/FBuildWorker/Worker/WorkerWindow.cpp +++ b/Code/Tools/FBuild/FBuildWorker/Worker/WorkerWindow.cpp @@ -158,10 +158,10 @@ WorkerWindow::WorkerWindow() #if defined( __WINDOWS__ ) // Display the window and minimize it if needed - if ( WorkerSettings::Get().GetStartMinimzed() ) + if ( WorkerSettings::Get().GetStartMinimized() ) { UpdateWindow( (HWND)GetHandle() ); - ToggleMinimized(); // minimze + ToggleMinimized(); // minimize } else { @@ -247,7 +247,7 @@ void WorkerWindow::Work() ToggleMinimized(); #endif - return true; // Stop window closeing (since we already handled it) + return true; // Stop window closing (since we already handled it) } // OnQuit diff --git a/Code/fbuild.bff b/Code/fbuild.bff index e6dd57161..97265ee49 100644 --- a/Code/fbuild.bff +++ b/Code/fbuild.bff @@ -99,13 +99,7 @@ Settings ] .Fuzzer_Config = [ - #if !USING_CLANG_3 - // These options are valid for Clang >= 6.0.0: - .CompilerOptions = ' -fsanitize=fuzzer-no-link' - #else - // These options are valid for Clang < 6.0.0: - .CompilerOptions = ' -fsanitize-coverage=trace-pc-guard,trace-cmp' - #endif + .CompilerOptions = ' -fsanitize=fuzzer-no-link' .CompilerOptionsC = .CompilerOptions ] @@ -461,10 +455,7 @@ Alias( 'Exes' ) .PlatformConfigs_x64 = { 'Debug', 'Analyze', 'Profile', 'Release', 'ASan', 'TSan' } .PlatformConfigs_x64Clang = { 'Debug', 'Analyze', 'Profile', 'Release', 'ASan', 'TSan' } .PlatformConfigs_x64Linux = { 'Debug', 'Profile', 'Release', 'ASan', 'TSan' } -.PlatformConfigs_x64ClangLinux = { 'Debug', 'Profile', 'Release' } -#if !USING_CLANG_3 - + { 'ASan', 'MSan', 'TSan' } -#endif +.PlatformConfigs_x64ClangLinux = { 'Debug', 'Profile', 'Release', 'ASan', 'MSan', 'TSan' } .PlatformConfigs_x64OSX = { 'Debug', 'Profile', 'Release' } #if CLANG_SUPPORTS_ARMOSX .PlatformConfigs_ARMOSX = { 'Debug', 'Profile', 'Release' } diff --git a/External/SDK/Clang/Linux/Clang10.bff b/External/SDK/Clang/Linux/Clang10.bff index 0d3e9534a..0253b222c 100644 --- a/External/SDK/Clang/Linux/Clang10.bff +++ b/External/SDK/Clang/Linux/Clang10.bff @@ -55,7 +55,6 @@ Compiler( 'Compiler-Clang10' ) // Warnings that need fixing or further investigation + ' -Wno-atomic-implicit-seq-cst' // implicit use of sequentially-consistent atomic may incur stronger memory barriers than necessary - + ' -Wno-c++17-extensions' // TODO: Use std-c++17 + ' -Wno-cast-align' // cast from '%s' to '%s' increases required alignment from 1 to 4 + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier + ' -Wno-deprecated-copy-dtor' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor diff --git a/External/SDK/Clang/Linux/Clang14.bff b/External/SDK/Clang/Linux/Clang14.bff index 614edc4a7..01126bc58 100644 --- a/External/SDK/Clang/Linux/Clang14.bff +++ b/External/SDK/Clang/Linux/Clang14.bff @@ -55,7 +55,6 @@ Compiler( 'Compiler-Clang14' ) // Warnings that need fixing or further investigation + ' -Wno-atomic-implicit-seq-cst' // implicit use of sequentially-consistent atomic may incur stronger memory barriers than necessary - + ' -Wno-c++17-extensions' // TODO: Use std-c++17 + ' -Wno-cast-align' // cast from '%s' to '%s' increases required alignment from 1 to 4 + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier + ' -Wno-deprecated-copy-dtor' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor diff --git a/External/SDK/Clang/Linux/Clang6.bff b/External/SDK/Clang/Linux/Clang6.bff index d756e8107..7183e4f85 100644 --- a/External/SDK/Clang/Linux/Clang6.bff +++ b/External/SDK/Clang/Linux/Clang6.bff @@ -55,7 +55,6 @@ Compiler( 'Compiler-Clang6' ) // Warnings that need fixing or further investigation + ' -Wno-atomic-implicit-seq-cst' // implicit use of sequentially-consistent atomic may incur stronger memory barriers than necessary - + ' -Wno-c++17-extensions' // TODO: Use std-c++17 + ' -Wno-cast-align' // cast from '%s' to '%s' increases required alignment from 1 to 4 + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier + ' -Wno-deprecated-copy-dtor' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor diff --git a/External/SDK/Clang/Linux/Clang_CI.bff b/External/SDK/Clang/Linux/Clang_CI.bff index 4c576d008..6a283409c 100644 --- a/External/SDK/Clang/Linux/Clang_CI.bff +++ b/External/SDK/Clang/Linux/Clang_CI.bff @@ -54,7 +54,6 @@ Compiler( 'Compiler-Clang' ) // Warnings that need fixing or further investigation + ' -Wno-atomic-implicit-seq-cst' // implicit use of sequentially-consistent atomic may incur stronger memory barriers than necessary - + ' -Wno-c++17-extensions' // TODO: Use std-c++17 + ' -Wno-cast-align' // cast from '%s' to '%s' increases required alignment from 1 to 4 + ' -Wno-cast-qual' // cast from 'const %s *' to '%s *' drops const qualifier + ' -Wno-deprecated-copy-dtor' // definition of implicit copy constructor for '%s' is deprecated because it has a user-declared destructor diff --git a/Scripts/PrepareRelease.py b/Scripts/PrepareRelease.py index 8bd4ebc4c..f262555d0 100644 --- a/Scripts/PrepareRelease.py +++ b/Scripts/PrepareRelease.py @@ -120,7 +120,7 @@ def prepare_output_folders(version): os.mkdir(OUTPUT_ROOT) if not os.path.exists(f'{OUTPUT_ROOT}/{version}'): os.mkdir(f'{OUTPUT_ROOT}/{version}') - print_ok() + print_ok(f'{OUTPUT_ROOT}/{version}') except: print_fail('FAIL', 'Directory creation failed') return False @@ -182,6 +182,7 @@ def get_files_recurse(base_path, out_files): file.endswith('.d') or \ file.endswith('.fdb') or \ file.endswith('.pyc') or \ + file == 'compile_commands.json' or \ file == 'fbuild_profile.json' or \ file == 'profile.json': continue;

VersionWindowsOS XLinuxSourceMarkup
v1.10x64x64x64Zip | + x64x64x64Zip | GitHub NotePad++ Visual Studio
VersionWindowsOS XLinuxSource
v1.11 x64x64 & ARMx64 & ARM x64 Zip | GitHub
VersionWindowsOS XLinuxSourceMarkup
v1.10 x64x64x64 & ARM x64 Zip | GitHub